1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 reg_class p0_reg(P0);
 1203 reg_class p1_reg(P1);
 1204 
 1205 // Singleton class for condition codes
 1206 reg_class int_flags(RFLAGS);
 1207 
 1208 %}
 1209 
 1210 //----------DEFINITION BLOCK---------------------------------------------------
 1211 // Define name --> value mappings to inform the ADLC of an integer valued name
 1212 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1213 // Format:
 1214 //        int_def  <name>         ( <int_value>, <expression>);
 1215 // Generated Code in ad_<arch>.hpp
 1216 //        #define  <name>   (<expression>)
 1217 //        // value == <int_value>
 1218 // Generated code in ad_<arch>.cpp adlc_verification()
 1219 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1220 //
 1221 
 1222 // we follow the ppc-aix port in using a simple cost model which ranks
 1223 // register operations as cheap, memory ops as more expensive and
 1224 // branches as most expensive. the first two have a low as well as a
 1225 // normal cost. huge cost appears to be a way of saying don't do
 1226 // something
 1227 
 1228 definitions %{
 1229   // The default cost (of a register move instruction).
 1230   int_def INSN_COST            (    100,     100);
 1231   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1232   int_def CALL_COST            (    200,     2 * INSN_COST);
 1233   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1234 %}
 1235 
 1236 
 1237 //----------SOURCE BLOCK-------------------------------------------------------
 1238 // This is a block of C++ code which provides values, functions, and
 1239 // definitions necessary in the rest of the architecture description
 1240 
 1241 source_hpp %{
 1242 
 1243 #include "asm/macroAssembler.hpp"
 1244 #include "gc/shared/barrierSetAssembler.hpp"
 1245 #include "gc/shared/cardTable.hpp"
 1246 #include "gc/shared/cardTableBarrierSet.hpp"
 1247 #include "gc/shared/collectedHeap.hpp"
 1248 #include "opto/addnode.hpp"
 1249 #include "opto/convertnode.hpp"
 1250 #include "runtime/objectMonitor.hpp"
 1251 
 1252 extern RegMask _ANY_REG32_mask;
 1253 extern RegMask _ANY_REG_mask;
 1254 extern RegMask _PTR_REG_mask;
 1255 extern RegMask _NO_SPECIAL_REG32_mask;
 1256 extern RegMask _NO_SPECIAL_REG_mask;
 1257 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1258 
 1259 class CallStubImpl {
 1260 
 1261   //--------------------------------------------------------------
 1262   //---<  Used for optimization in Compile::shorten_branches  >---
 1263   //--------------------------------------------------------------
 1264 
 1265  public:
 1266   // Size of call trampoline stub.
 1267   static uint size_call_trampoline() {
 1268     return 0; // no call trampolines on this platform
 1269   }
 1270 
 1271   // number of relocations needed by a call trampoline stub
 1272   static uint reloc_call_trampoline() {
 1273     return 0; // no call trampolines on this platform
 1274   }
 1275 };
 1276 
 1277 class HandlerImpl {
 1278 
 1279  public:
 1280 
 1281   static int emit_exception_handler(CodeBuffer &cbuf);
 1282   static int emit_deopt_handler(CodeBuffer& cbuf);
 1283 
 1284   static uint size_exception_handler() {
 1285     return MacroAssembler::far_branch_size();
 1286   }
 1287 
 1288   static uint size_deopt_handler() {
 1289     // count one adr and one far branch instruction
 1290     return 4 * NativeInstruction::instruction_size;
 1291   }
 1292 };
 1293 
 1294 class Node::PD {
 1295 public:
 1296   enum NodeFlags {
 1297     _last_flag = Node::_last_flag
 1298   };
 1299 };
 1300 
 1301   bool is_CAS(int opcode, bool maybe_volatile);
 1302 
 1303   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1304 
 1305   bool unnecessary_acquire(const Node *barrier);
 1306   bool needs_acquiring_load(const Node *load);
 1307 
 1308   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1309 
 1310   bool unnecessary_release(const Node *barrier);
 1311   bool unnecessary_volatile(const Node *barrier);
 1312   bool needs_releasing_store(const Node *store);
 1313 
 1314   // predicate controlling translation of CompareAndSwapX
 1315   bool needs_acquiring_load_exclusive(const Node *load);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 #define __ _masm.
 1732 
 1733 // advance declarations for helper functions to convert register
 1734 // indices to register objects
 1735 
 1736 // the ad file has to provide implementations of certain methods
 1737 // expected by the generic code
 1738 //
 1739 // REQUIRED FUNCTIONALITY
 1740 
 1741 //=============================================================================
 1742 
 1743 // !!!!! Special hack to get all types of calls to specify the byte offset
 1744 //       from the start of the call to the point where the return address
 1745 //       will point.
 1746 
 1747 int MachCallStaticJavaNode::ret_addr_offset()
 1748 {
 1749   // call should be a simple bl
 1750   int off = 4;
 1751   return off;
 1752 }
 1753 
 1754 int MachCallDynamicJavaNode::ret_addr_offset()
 1755 {
 1756   return 16; // movz, movk, movk, bl
 1757 }
 1758 
 1759 int MachCallRuntimeNode::ret_addr_offset() {
 1760   // for generated stubs the call will be
 1761   //   bl(addr)
 1762   // or with far branches
 1763   //   bl(trampoline_stub)
 1764   // for real runtime callouts it will be six instructions
 1765   // see aarch64_enc_java_to_runtime
 1766   //   adr(rscratch2, retaddr)
 1767   //   lea(rscratch1, RuntimeAddress(addr)
 1768   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1769   //   blr(rscratch1)
 1770   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1771   if (cb) {
 1772     return 1 * NativeInstruction::instruction_size;
 1773   } else {
 1774     return 6 * NativeInstruction::instruction_size;
 1775   }
 1776 }
 1777 
 1778 int MachCallNativeNode::ret_addr_offset() {
 1779   // This is implemented using aarch64_enc_java_to_runtime as above.
 1780   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1781   if (cb) {
 1782     return 1 * NativeInstruction::instruction_size;
 1783   } else {
 1784     return 6 * NativeInstruction::instruction_size;
 1785   }
 1786 }
 1787 
 1788 //=============================================================================
 1789 
 1790 #ifndef PRODUCT
 1791 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1792   st->print("BREAKPOINT");
 1793 }
 1794 #endif
 1795 
 1796 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1797   C2_MacroAssembler _masm(&cbuf);
 1798   __ brk(0);
 1799 }
 1800 
 1801 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1802   return MachNode::size(ra_);
 1803 }
 1804 
 1805 //=============================================================================
 1806 
 1807 #ifndef PRODUCT
 1808   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1809     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1810   }
 1811 #endif
 1812 
 1813   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1814     C2_MacroAssembler _masm(&cbuf);
 1815     for (int i = 0; i < _count; i++) {
 1816       __ nop();
 1817     }
 1818   }
 1819 
 1820   uint MachNopNode::size(PhaseRegAlloc*) const {
 1821     return _count * NativeInstruction::instruction_size;
 1822   }
 1823 
 1824 //=============================================================================
 1825 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1826 
 1827 int ConstantTable::calculate_table_base_offset() const {
 1828   return 0;  // absolute addressing, no offset
 1829 }
 1830 
 1831 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1832 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1833   ShouldNotReachHere();
 1834 }
 1835 
 1836 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1837   // Empty encoding
 1838 }
 1839 
 1840 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1841   return 0;
 1842 }
 1843 
 1844 #ifndef PRODUCT
 1845 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1846   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1847 }
 1848 #endif
 1849 
 1850 #ifndef PRODUCT
 1851 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1852   Compile* C = ra_->C;
 1853 
 1854   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1855 
 1856   if (C->output()->need_stack_bang(framesize))
 1857     st->print("# stack bang size=%d\n\t", framesize);
 1858 
 1859   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1860     st->print("sub  sp, sp, #%d\n\t", framesize);
 1861     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1862     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1863   } else {
 1864     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1865     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1866     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1867     st->print("sub  sp, sp, rscratch1");
 1868   }
 1869   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1870     st->print("\n\t");
 1871     st->print("ldr  rscratch1, [guard]\n\t");
 1872     st->print("dmb ishld\n\t");
 1873     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1874     st->print("cmp  rscratch1, rscratch2\n\t");
 1875     st->print("b.eq skip");
 1876     st->print("\n\t");
 1877     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1878     st->print("b skip\n\t");
 1879     st->print("guard: int\n\t");
 1880     st->print("\n\t");
 1881     st->print("skip:\n\t");
 1882   }
 1883 }
 1884 #endif
 1885 
 1886 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1887   Compile* C = ra_->C;
 1888   C2_MacroAssembler _masm(&cbuf);
 1889 
 1890   // n.b. frame size includes space for return pc and rfp
 1891   const int framesize = C->output()->frame_size_in_bytes();
 1892 
 1893   // insert a nop at the start of the prolog so we can patch in a
 1894   // branch if we need to invalidate the method later
 1895   __ nop();
 1896 
 1897   if (C->clinit_barrier_on_entry()) {
 1898     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1899 
 1900     Label L_skip_barrier;
 1901 
 1902     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1903     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1904     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1905     __ bind(L_skip_barrier);
 1906   }
 1907 
 1908   if (C->max_vector_size() > 0) {
 1909     __ reinitialize_ptrue();
 1910   }
 1911 
 1912   int bangsize = C->output()->bang_size_in_bytes();
 1913   if (C->output()->need_stack_bang(bangsize))
 1914     __ generate_stack_overflow_check(bangsize);
 1915 
 1916   __ build_frame(framesize);
 1917 
 1918   if (C->stub_function() == NULL) {
 1919     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1920     bs->nmethod_entry_barrier(&_masm);
 1921   }
 1922 
 1923   if (VerifyStackAtCalls) {
 1924     Unimplemented();
 1925   }
 1926 
 1927   C->output()->set_frame_complete(cbuf.insts_size());
 1928 
 1929   if (C->has_mach_constant_base_node()) {
 1930     // NOTE: We set the table base offset here because users might be
 1931     // emitted before MachConstantBaseNode.
 1932     ConstantTable& constant_table = C->output()->constant_table();
 1933     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1934   }
 1935 }
 1936 
 1937 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1938 {
 1939   return MachNode::size(ra_); // too many variables; just compute it
 1940                               // the hard way
 1941 }
 1942 
 1943 int MachPrologNode::reloc() const
 1944 {
 1945   return 0;
 1946 }
 1947 
 1948 //=============================================================================
 1949 
 1950 #ifndef PRODUCT
 1951 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1952   Compile* C = ra_->C;
 1953   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1954 
 1955   st->print("# pop frame %d\n\t",framesize);
 1956 
 1957   if (framesize == 0) {
 1958     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1959   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1960     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, #%d\n\t", framesize);
 1962   } else {
 1963     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1964     st->print("add  sp, sp, rscratch1\n\t");
 1965     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1966   }
 1967 
 1968   if (do_polling() && C->is_method_compilation()) {
 1969     st->print("# test polling word\n\t");
 1970     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1971     st->print("cmp  sp, rscratch1\n\t");
 1972     st->print("bhi #slow_path");
 1973   }
 1974 }
 1975 #endif
 1976 
 1977 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1978   Compile* C = ra_->C;
 1979   C2_MacroAssembler _masm(&cbuf);
 1980   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1981 
 1982   __ remove_frame(framesize);
 1983 
 1984   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1985     __ reserved_stack_check();
 1986   }
 1987 
 1988   if (do_polling() && C->is_method_compilation()) {
 1989     Label dummy_label;
 1990     Label* code_stub = &dummy_label;
 1991     if (!C->output()->in_scratch_emit_size()) {
 1992       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1993     }
 1994     __ relocate(relocInfo::poll_return_type);
 1995     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1996   }
 1997 }
 1998 
 1999 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2000   // Variable size. Determine dynamically.
 2001   return MachNode::size(ra_);
 2002 }
 2003 
 2004 int MachEpilogNode::reloc() const {
 2005   // Return number of relocatable values contained in this instruction.
 2006   return 1; // 1 for polling page.
 2007 }
 2008 
 2009 const Pipeline * MachEpilogNode::pipeline() const {
 2010   return MachNode::pipeline_class();
 2011 }
 2012 
 2013 //=============================================================================
 2014 
 2015 // Figure out which register class each belongs in: rc_int, rc_float or
 2016 // rc_stack.
 2017 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2018 
 2019 static enum RC rc_class(OptoReg::Name reg) {
 2020 
 2021   if (reg == OptoReg::Bad) {
 2022     return rc_bad;
 2023   }
 2024 
 2025   // we have 32 int registers * 2 halves
 2026   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2027 
 2028   if (reg < slots_of_int_registers) {
 2029     return rc_int;
 2030   }
 2031 
 2032   // we have 32 float register * 8 halves
 2033   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2034   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2035     return rc_float;
 2036   }
 2037 
 2038   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2039   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2040     return rc_predicate;
 2041   }
 2042 
 2043   // Between predicate regs & stack is the flags.
 2044   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2045 
 2046   return rc_stack;
 2047 }
 2048 
 2049 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2050   Compile* C = ra_->C;
 2051 
 2052   // Get registers to move.
 2053   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2054   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2055   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2056   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2057 
 2058   enum RC src_hi_rc = rc_class(src_hi);
 2059   enum RC src_lo_rc = rc_class(src_lo);
 2060   enum RC dst_hi_rc = rc_class(dst_hi);
 2061   enum RC dst_lo_rc = rc_class(dst_lo);
 2062 
 2063   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2064 
 2065   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2066     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2067            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2068            "expected aligned-adjacent pairs");
 2069   }
 2070 
 2071   if (src_lo == dst_lo && src_hi == dst_hi) {
 2072     return 0;            // Self copy, no move.
 2073   }
 2074 
 2075   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2076               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2077   int src_offset = ra_->reg2offset(src_lo);
 2078   int dst_offset = ra_->reg2offset(dst_lo);
 2079 
 2080   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2081     uint ireg = ideal_reg();
 2082     if (ireg == Op_VecA && cbuf) {
 2083       C2_MacroAssembler _masm(cbuf);
 2084       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2085       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2086         // stack->stack
 2087         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2088                                                 sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2090         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2091                             sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2093         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2094                               sve_vector_reg_size_in_bytes);
 2095       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2096         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2097                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2098                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2099       } else {
 2100         ShouldNotReachHere();
 2101       }
 2102     } else if (cbuf) {
 2103       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2104       C2_MacroAssembler _masm(cbuf);
 2105       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2106       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2107         // stack->stack
 2108         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2109         if (ireg == Op_VecD) {
 2110           __ unspill(rscratch1, true, src_offset);
 2111           __ spill(rscratch1, true, dst_offset);
 2112         } else {
 2113           __ spill_copy128(src_offset, dst_offset);
 2114         }
 2115       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2116         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2117                ireg == Op_VecD ? __ T8B : __ T16B,
 2118                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2119       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2120         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2121                  ireg == Op_VecD ? __ D : __ Q,
 2122                  ra_->reg2offset(dst_lo));
 2123       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2124         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2125                    ireg == Op_VecD ? __ D : __ Q,
 2126                    ra_->reg2offset(src_lo));
 2127       } else {
 2128         ShouldNotReachHere();
 2129       }
 2130     }
 2131   } else if (cbuf) {
 2132     C2_MacroAssembler _masm(cbuf);
 2133     switch (src_lo_rc) {
 2134     case rc_int:
 2135       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2136         if (is64) {
 2137             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2138                    as_Register(Matcher::_regEncode[src_lo]));
 2139         } else {
 2140             C2_MacroAssembler _masm(cbuf);
 2141             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2142                     as_Register(Matcher::_regEncode[src_lo]));
 2143         }
 2144       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2145         if (is64) {
 2146             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         } else {
 2149             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2150                      as_Register(Matcher::_regEncode[src_lo]));
 2151         }
 2152       } else {                    // gpr --> stack spill
 2153         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2154         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2155       }
 2156       break;
 2157     case rc_float:
 2158       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2159         if (is64) {
 2160             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2164                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2167         if (is64) {
 2168             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         } else {
 2171             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2172                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2173         }
 2174       } else {                    // fpr --> stack spill
 2175         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2176         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2177                  is64 ? __ D : __ S, dst_offset);
 2178       }
 2179       break;
 2180     case rc_stack:
 2181       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2182         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2183       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2184         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2185                    is64 ? __ D : __ S, src_offset);
 2186       } else if (dst_lo_rc == rc_predicate) {
 2187         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2188                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2189       } else {                    // stack --> stack copy
 2190         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2191         if (ideal_reg() == Op_RegVectMask) {
 2192           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2193                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2194         } else {
 2195           __ unspill(rscratch1, is64, src_offset);
 2196           __ spill(rscratch1, is64, dst_offset);
 2197         }
 2198       }
 2199       break;
 2200     case rc_predicate:
 2201       if (dst_lo_rc == rc_predicate) {
 2202         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2203       } else if (dst_lo_rc == rc_stack) {
 2204         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2205                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2206       } else {
 2207         assert(false, "bad src and dst rc_class combination.");
 2208         ShouldNotReachHere();
 2209       }
 2210       break;
 2211     default:
 2212       assert(false, "bad rc_class for spill");
 2213       ShouldNotReachHere();
 2214     }
 2215   }
 2216 
 2217   if (st) {
 2218     st->print("spill ");
 2219     if (src_lo_rc == rc_stack) {
 2220       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2221     } else {
 2222       st->print("%s -> ", Matcher::regName[src_lo]);
 2223     }
 2224     if (dst_lo_rc == rc_stack) {
 2225       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2226     } else {
 2227       st->print("%s", Matcher::regName[dst_lo]);
 2228     }
 2229     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2230       int vsize = 0;
 2231       switch (ideal_reg()) {
 2232       case Op_VecD:
 2233         vsize = 64;
 2234         break;
 2235       case Op_VecX:
 2236         vsize = 128;
 2237         break;
 2238       case Op_VecA:
 2239         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2240         break;
 2241       default:
 2242         assert(false, "bad register type for spill");
 2243         ShouldNotReachHere();
 2244       }
 2245       st->print("\t# vector spill size = %d", vsize);
 2246     } else if (ideal_reg() == Op_RegVectMask) {
 2247       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2248       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2249       st->print("\t# predicate spill size = %d", vsize);
 2250     } else {
 2251       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2252     }
 2253   }
 2254 
 2255   return 0;
 2256 
 2257 }
 2258 
 2259 #ifndef PRODUCT
 2260 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2261   if (!ra_)
 2262     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2263   else
 2264     implementation(NULL, ra_, false, st);
 2265 }
 2266 #endif
 2267 
 2268 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2269   implementation(&cbuf, ra_, false, NULL);
 2270 }
 2271 
 2272 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2273   return MachNode::size(ra_);
 2274 }
 2275 
 2276 //=============================================================================
 2277 
 2278 #ifndef PRODUCT
 2279 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2280   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2281   int reg = ra_->get_reg_first(this);
 2282   st->print("add %s, rsp, #%d]\t# box lock",
 2283             Matcher::regName[reg], offset);
 2284 }
 2285 #endif
 2286 
 2287 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2288   C2_MacroAssembler _masm(&cbuf);
 2289 
 2290   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2291   int reg    = ra_->get_encode(this);
 2292 
 2293   // This add will handle any 24-bit signed offset. 24 bits allows an
 2294   // 8 megabyte stack frame.
 2295   __ add(as_Register(reg), sp, offset);
 2296 }
 2297 
 2298 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2299   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2300   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2301 
 2302   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2303     return NativeInstruction::instruction_size;
 2304   } else {
 2305     return 2 * NativeInstruction::instruction_size;
 2306   }
 2307 }
 2308 
 2309 //=============================================================================
 2310 
 2311 #ifndef PRODUCT
 2312 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2313 {
 2314   st->print_cr("# MachUEPNode");
 2315   if (UseCompressedClassPointers) {
 2316     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2317     if (CompressedKlassPointers::shift() != 0) {
 2318       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2319     }
 2320   } else {
 2321    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2322   }
 2323   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2324   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2325 }
 2326 #endif
 2327 
 2328 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2329 {
 2330   // This is the unverified entry point.
 2331   C2_MacroAssembler _masm(&cbuf);
 2332 
 2333   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2334   Label skip;
 2335   // TODO
 2336   // can we avoid this skip and still use a reloc?
 2337   __ br(Assembler::EQ, skip);
 2338   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2339   __ bind(skip);
 2340 }
 2341 
 2342 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2343 {
 2344   return MachNode::size(ra_);
 2345 }
 2346 
 2347 // REQUIRED EMIT CODE
 2348 
 2349 //=============================================================================
 2350 
 2351 // Emit exception handler code.
 2352 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2353 {
 2354   // mov rscratch1 #exception_blob_entry_point
 2355   // br rscratch1
 2356   // Note that the code buffer's insts_mark is always relative to insts.
 2357   // That's why we must use the macroassembler to generate a handler.
 2358   C2_MacroAssembler _masm(&cbuf);
 2359   address base = __ start_a_stub(size_exception_handler());
 2360   if (base == NULL) {
 2361     ciEnv::current()->record_failure("CodeCache is full");
 2362     return 0;  // CodeBuffer::expand failed
 2363   }
 2364   int offset = __ offset();
 2365   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2366   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2367   __ end_a_stub();
 2368   return offset;
 2369 }
 2370 
 2371 // Emit deopt handler code.
 2372 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2373 {
 2374   // Note that the code buffer's insts_mark is always relative to insts.
 2375   // That's why we must use the macroassembler to generate a handler.
 2376   C2_MacroAssembler _masm(&cbuf);
 2377   address base = __ start_a_stub(size_deopt_handler());
 2378   if (base == NULL) {
 2379     ciEnv::current()->record_failure("CodeCache is full");
 2380     return 0;  // CodeBuffer::expand failed
 2381   }
 2382   int offset = __ offset();
 2383 
 2384   __ adr(lr, __ pc());
 2385   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2386 
 2387   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2388   __ end_a_stub();
 2389   return offset;
 2390 }
 2391 
 2392 // REQUIRED MATCHER CODE
 2393 
 2394 //=============================================================================
 2395 
 2396 const bool Matcher::match_rule_supported(int opcode) {
 2397   if (!has_match_rule(opcode))
 2398     return false;
 2399 
 2400   bool ret_value = true;
 2401   switch (opcode) {
 2402     case Op_OnSpinWait:
 2403       return VM_Version::supports_on_spin_wait();
 2404     case Op_CacheWB:
 2405     case Op_CacheWBPreSync:
 2406     case Op_CacheWBPostSync:
 2407       if (!VM_Version::supports_data_cache_line_flush()) {
 2408         ret_value = false;
 2409       }
 2410       break;
 2411     case Op_LoadVectorMasked:
 2412     case Op_StoreVectorMasked:
 2413     case Op_LoadVectorGatherMasked:
 2414     case Op_StoreVectorScatterMasked:
 2415     case Op_MaskAll:
 2416     case Op_AndVMask:
 2417     case Op_OrVMask:
 2418     case Op_XorVMask:
 2419       if (UseSVE == 0) {
 2420         ret_value = false;
 2421       }
 2422       break;
 2423   }
 2424 
 2425   return ret_value; // Per default match rules are supported.
 2426 }
 2427 
 2428 // Identify extra cases that we might want to provide match rules for vector nodes and
 2429 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2430 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2431   if (!match_rule_supported(opcode)) {
 2432     return false;
 2433   }
 2434   int bit_size = vlen * type2aelembytes(bt) * 8;
 2435   if (UseSVE == 0 && bit_size > 128) {
 2436     return false;
 2437   }
 2438   if (UseSVE > 0) {
 2439     return op_sve_supported(opcode, vlen, bt);
 2440   } else { // NEON
 2441     // Special cases
 2442     switch (opcode) {
 2443     case Op_VectorMaskCmp:
 2444       if (vlen < 2 || bit_size < 64) {
 2445         return false;
 2446       }
 2447       break;
 2448     case Op_MulAddVS2VI:
 2449       if (bit_size < 128) {
 2450         return false;
 2451       }
 2452       break;
 2453     case Op_MulVL:
 2454       return false;
 2455     case Op_VectorLoadShuffle:
 2456     case Op_VectorRearrange:
 2457       if (vlen < 4) {
 2458         return false;
 2459       }
 2460       break;
 2461     case Op_LoadVectorGather:
 2462     case Op_StoreVectorScatter:
 2463       return false;
 2464     default:
 2465       break;
 2466     }
 2467   }
 2468   return vector_size_supported(bt, vlen);
 2469 }
 2470 
 2471 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2472   // Only SVE supports masked operations.
 2473   if (UseSVE == 0) {
 2474     return false;
 2475   }
 2476   return match_rule_supported(opcode) &&
 2477          masked_op_sve_supported(opcode, vlen, bt);
 2478 }
 2479 
 2480 const RegMask* Matcher::predicate_reg_mask(void) {
 2481   return &_PR_REG_mask;
 2482 }
 2483 
 2484 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2485   return new TypeVectMask(elemTy, length);
 2486 }
 2487 
 2488 // Vector calling convention not yet implemented.
 2489 const bool Matcher::supports_vector_calling_convention(void) {
 2490   return false;
 2491 }
 2492 
 2493 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2494   Unimplemented();
 2495   return OptoRegPair(0, 0);
 2496 }
 2497 
 2498 // Is this branch offset short enough that a short branch can be used?
 2499 //
 2500 // NOTE: If the platform does not provide any short branch variants, then
 2501 //       this method should return false for offset 0.
 2502 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2503   // The passed offset is relative to address of the branch.
 2504 
 2505   return (-32768 <= offset && offset < 32768);
 2506 }
 2507 
 2508 // Vector width in bytes.
 2509 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2510   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2511   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2512   // Minimum 2 values in vector
 2513   if (size < 2*type2aelembytes(bt)) size = 0;
 2514   // But never < 4
 2515   if (size < 4) size = 0;
 2516   return size;
 2517 }
 2518 
 2519 // Limits on vector size (number of elements) loaded into vector.
 2520 const int Matcher::max_vector_size(const BasicType bt) {
 2521   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2522 }
 2523 
 2524 const int Matcher::min_vector_size(const BasicType bt) {
 2525   int max_size = max_vector_size(bt);
 2526   // Limit the min vector size to 8 bytes.
 2527   int size = 8 / type2aelembytes(bt);
 2528   if (bt == T_BYTE) {
 2529     // To support vector api shuffle/rearrange.
 2530     size = 4;
 2531   } else if (bt == T_BOOLEAN) {
 2532     // To support vector api load/store mask.
 2533     size = 2;
 2534   }
 2535   if (size < 2) size = 2;
 2536   return MIN2(size, max_size);
 2537 }
 2538 
 2539 // Actual max scalable vector register length.
 2540 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2541   return Matcher::max_vector_size(bt);
 2542 }
 2543 
 2544 // Vector ideal reg.
 2545 const uint Matcher::vector_ideal_reg(int len) {
 2546   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2547     return Op_VecA;
 2548   }
 2549   switch(len) {
 2550     // For 16-bit/32-bit mask vector, reuse VecD.
 2551     case  2:
 2552     case  4:
 2553     case  8: return Op_VecD;
 2554     case 16: return Op_VecX;
 2555   }
 2556   ShouldNotReachHere();
 2557   return 0;
 2558 }
 2559 
 2560 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2561   ShouldNotReachHere(); // generic vector operands not supported
 2562   return NULL;
 2563 }
 2564 
 2565 bool Matcher::is_reg2reg_move(MachNode* m) {
 2566   ShouldNotReachHere();  // generic vector operands not supported
 2567   return false;
 2568 }
 2569 
 2570 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2571   ShouldNotReachHere();  // generic vector operands not supported
 2572   return false;
 2573 }
 2574 
 2575 // Return whether or not this register is ever used as an argument.
 2576 // This function is used on startup to build the trampoline stubs in
 2577 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2578 // call in the trampoline, and arguments in those registers not be
 2579 // available to the callee.
 2580 bool Matcher::can_be_java_arg(int reg)
 2581 {
 2582   return
 2583     reg ==  R0_num || reg == R0_H_num ||
 2584     reg ==  R1_num || reg == R1_H_num ||
 2585     reg ==  R2_num || reg == R2_H_num ||
 2586     reg ==  R3_num || reg == R3_H_num ||
 2587     reg ==  R4_num || reg == R4_H_num ||
 2588     reg ==  R5_num || reg == R5_H_num ||
 2589     reg ==  R6_num || reg == R6_H_num ||
 2590     reg ==  R7_num || reg == R7_H_num ||
 2591     reg ==  V0_num || reg == V0_H_num ||
 2592     reg ==  V1_num || reg == V1_H_num ||
 2593     reg ==  V2_num || reg == V2_H_num ||
 2594     reg ==  V3_num || reg == V3_H_num ||
 2595     reg ==  V4_num || reg == V4_H_num ||
 2596     reg ==  V5_num || reg == V5_H_num ||
 2597     reg ==  V6_num || reg == V6_H_num ||
 2598     reg ==  V7_num || reg == V7_H_num;
 2599 }
 2600 
 2601 bool Matcher::is_spillable_arg(int reg)
 2602 {
 2603   return can_be_java_arg(reg);
 2604 }
 2605 
 2606 uint Matcher::int_pressure_limit()
 2607 {
 2608   // JDK-8183543: When taking the number of available registers as int
 2609   // register pressure threshold, the jtreg test:
 2610   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2611   // failed due to C2 compilation failure with
 2612   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2613   //
 2614   // A derived pointer is live at CallNode and then is flagged by RA
 2615   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2616   // derived pointers and lastly fail to spill after reaching maximum
 2617   // number of iterations. Lowering the default pressure threshold to
 2618   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2619   // a high register pressure area of the code so that split_DEF can
 2620   // generate DefinitionSpillCopy for the derived pointer.
 2621   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2622   if (!PreserveFramePointer) {
 2623     // When PreserveFramePointer is off, frame pointer is allocatable,
 2624     // but different from other SOC registers, it is excluded from
 2625     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2626     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2627     // See check_pressure_at_fatproj().
 2628     default_int_pressure_threshold--;
 2629   }
 2630   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2631 }
 2632 
 2633 uint Matcher::float_pressure_limit()
 2634 {
 2635   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2636   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2637 }
 2638 
 2639 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2640   return false;
 2641 }
 2642 
 2643 RegMask Matcher::divI_proj_mask() {
 2644   ShouldNotReachHere();
 2645   return RegMask();
 2646 }
 2647 
 2648 // Register for MODI projection of divmodI.
 2649 RegMask Matcher::modI_proj_mask() {
 2650   ShouldNotReachHere();
 2651   return RegMask();
 2652 }
 2653 
 2654 // Register for DIVL projection of divmodL.
 2655 RegMask Matcher::divL_proj_mask() {
 2656   ShouldNotReachHere();
 2657   return RegMask();
 2658 }
 2659 
 2660 // Register for MODL projection of divmodL.
 2661 RegMask Matcher::modL_proj_mask() {
 2662   ShouldNotReachHere();
 2663   return RegMask();
 2664 }
 2665 
 2666 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2667   return FP_REG_mask();
 2668 }
 2669 
 2670 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2671   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2672     Node* u = addp->fast_out(i);
 2673     if (u->is_LoadStore()) {
 2674       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2675       // instructions) only take register indirect as an operand, so
 2676       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2677       // must fail.
 2678       return false;
 2679     }
 2680     if (u->is_Mem()) {
 2681       int opsize = u->as_Mem()->memory_size();
 2682       assert(opsize > 0, "unexpected memory operand size");
 2683       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2684         return false;
 2685       }
 2686     }
 2687   }
 2688   return true;
 2689 }
 2690 
 2691 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2692   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2693     return false;
 2694   }
 2695   Node* imm_node = replicate_node->in(1);
 2696   if (!imm_node->is_Con()) {
 2697     return false;
 2698   }
 2699 
 2700   const Type* t = imm_node->bottom_type();
 2701   if (!(t->isa_int() || t->isa_long())) {
 2702     return false;
 2703   }
 2704 
 2705   switch (binary_node->Opcode()) {
 2706   case Op_AndV:
 2707   case Op_OrV:
 2708   case Op_XorV: {
 2709     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2710     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2711     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2712   }
 2713   case Op_AddVB:
 2714     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2715   case Op_AddVS:
 2716   case Op_AddVI:
 2717     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2718   case Op_AddVL:
 2719     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2720   default:
 2721     return false;
 2722   }
 2723 }
 2724 
 2725 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2726   if (n != NULL && m != NULL) {
 2727     return can_combine_with_imm(n, m);
 2728   }
 2729   return false;
 2730 }
 2731 
 2732 // Should the matcher clone input 'm' of node 'n'?
 2733 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2734   // ShiftV src (ShiftCntV con)
 2735   // StoreVector (VectorStoreMask src)
 2736   // Binary src (Replicate con)
 2737   if (is_vshift_con_pattern(n, m) ||
 2738       (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector) ||
 2739       is_vector_arith_imm_pattern(n, m)) {
 2740     mstack.push(m, Visit);
 2741     return true;
 2742   }
 2743 
 2744   return false;
 2745 }
 2746 
 2747 // Should the Matcher clone shifts on addressing modes, expecting them
 2748 // to be subsumed into complex addressing expressions or compute them
 2749 // into registers?
 2750 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2751   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2752     return true;
 2753   }
 2754 
 2755   Node *off = m->in(AddPNode::Offset);
 2756   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2757       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2758       // Are there other uses besides address expressions?
 2759       !is_visited(off)) {
 2760     address_visited.set(off->_idx); // Flag as address_visited
 2761     mstack.push(off->in(2), Visit);
 2762     Node *conv = off->in(1);
 2763     if (conv->Opcode() == Op_ConvI2L &&
 2764         // Are there other uses besides address expressions?
 2765         !is_visited(conv)) {
 2766       address_visited.set(conv->_idx); // Flag as address_visited
 2767       mstack.push(conv->in(1), Pre_Visit);
 2768     } else {
 2769       mstack.push(conv, Pre_Visit);
 2770     }
 2771     address_visited.test_set(m->_idx); // Flag as address_visited
 2772     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2773     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2774     return true;
 2775   } else if (off->Opcode() == Op_ConvI2L &&
 2776              // Are there other uses besides address expressions?
 2777              !is_visited(off)) {
 2778     address_visited.test_set(m->_idx); // Flag as address_visited
 2779     address_visited.set(off->_idx); // Flag as address_visited
 2780     mstack.push(off->in(1), Pre_Visit);
 2781     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2782     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2783     return true;
 2784   }
 2785   return false;
 2786 }
 2787 
 2788 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2789   C2_MacroAssembler _masm(&cbuf);                                       \
 2790   {                                                                     \
 2791     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2792     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2793     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2794     __ INSN(REG, as_Register(BASE));                                    \
 2795   }
 2796 
 2797 
 2798 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2799   {
 2800     Address::extend scale;
 2801 
 2802     // Hooboy, this is fugly.  We need a way to communicate to the
 2803     // encoder that the index needs to be sign extended, so we have to
 2804     // enumerate all the cases.
 2805     switch (opcode) {
 2806     case INDINDEXSCALEDI2L:
 2807     case INDINDEXSCALEDI2LN:
 2808     case INDINDEXI2L:
 2809     case INDINDEXI2LN:
 2810       scale = Address::sxtw(size);
 2811       break;
 2812     default:
 2813       scale = Address::lsl(size);
 2814     }
 2815 
 2816     if (index == -1) {
 2817       return Address(base, disp);
 2818     } else {
 2819       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2820       return Address(base, as_Register(index), scale);
 2821     }
 2822   }
 2823 
 2824 
 2825 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2826 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2827 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2828 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2829                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2830 
 2831   // Used for all non-volatile memory accesses.  The use of
 2832   // $mem->opcode() to discover whether this pattern uses sign-extended
 2833   // offsets is something of a kludge.
 2834   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2835                         Register reg, int opcode,
 2836                         Register base, int index, int scale, int disp,
 2837                         int size_in_memory)
 2838   {
 2839     Address addr = mem2address(opcode, base, index, scale, disp);
 2840     if (addr.getMode() == Address::base_plus_offset) {
 2841       /* If we get an out-of-range offset it is a bug in the compiler,
 2842          so we assert here. */
 2843       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2844              "c2 compiler bug");
 2845       /* Fix up any out-of-range offsets. */
 2846       assert_different_registers(rscratch1, base);
 2847       assert_different_registers(rscratch1, reg);
 2848       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2849     }
 2850     (masm.*insn)(reg, addr);
 2851   }
 2852 
 2853   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2854                         FloatRegister reg, int opcode,
 2855                         Register base, int index, int size, int disp,
 2856                         int size_in_memory)
 2857   {
 2858     Address::extend scale;
 2859 
 2860     switch (opcode) {
 2861     case INDINDEXSCALEDI2L:
 2862     case INDINDEXSCALEDI2LN:
 2863       scale = Address::sxtw(size);
 2864       break;
 2865     default:
 2866       scale = Address::lsl(size);
 2867     }
 2868 
 2869     if (index == -1) {
 2870       /* If we get an out-of-range offset it is a bug in the compiler,
 2871          so we assert here. */
 2872       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2873       /* Fix up any out-of-range offsets. */
 2874       assert_different_registers(rscratch1, base);
 2875       Address addr = Address(base, disp);
 2876       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2877       (masm.*insn)(reg, addr);
 2878     } else {
 2879       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2880       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2881     }
 2882   }
 2883 
 2884   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2885                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2886                         int opcode, Register base, int index, int size, int disp)
 2887   {
 2888     if (index == -1) {
 2889       (masm.*insn)(reg, T, Address(base, disp));
 2890     } else {
 2891       assert(disp == 0, "unsupported address mode");
 2892       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2893     }
 2894   }
 2895 
 2896 %}
 2897 
 2898 
 2899 
 2900 //----------ENCODING BLOCK-----------------------------------------------------
 2901 // This block specifies the encoding classes used by the compiler to
 2902 // output byte streams.  Encoding classes are parameterized macros
 2903 // used by Machine Instruction Nodes in order to generate the bit
 2904 // encoding of the instruction.  Operands specify their base encoding
 2905 // interface with the interface keyword.  There are currently
 2906 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2907 // COND_INTER.  REG_INTER causes an operand to generate a function
 2908 // which returns its register number when queried.  CONST_INTER causes
 2909 // an operand to generate a function which returns the value of the
 2910 // constant when queried.  MEMORY_INTER causes an operand to generate
 2911 // four functions which return the Base Register, the Index Register,
 2912 // the Scale Value, and the Offset Value of the operand when queried.
 2913 // COND_INTER causes an operand to generate six functions which return
 2914 // the encoding code (ie - encoding bits for the instruction)
 2915 // associated with each basic boolean condition for a conditional
 2916 // instruction.
 2917 //
 2918 // Instructions specify two basic values for encoding.  Again, a
 2919 // function is available to check if the constant displacement is an
 2920 // oop. They use the ins_encode keyword to specify their encoding
 2921 // classes (which must be a sequence of enc_class names, and their
 2922 // parameters, specified in the encoding block), and they use the
 2923 // opcode keyword to specify, in order, their primary, secondary, and
 2924 // tertiary opcode.  Only the opcode sections which a particular
 2925 // instruction needs for encoding need to be specified.
 2926 encode %{
 2927   // Build emit functions for each basic byte or larger field in the
 2928   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2929   // from C++ code in the enc_class source block.  Emit functions will
 2930   // live in the main source block for now.  In future, we can
 2931   // generalize this by adding a syntax that specifies the sizes of
 2932   // fields in an order, so that the adlc can build the emit functions
 2933   // automagically
 2934 
 2935   // catch all for unimplemented encodings
 2936   enc_class enc_unimplemented %{
 2937     C2_MacroAssembler _masm(&cbuf);
 2938     __ unimplemented("C2 catch all");
 2939   %}
 2940 
 2941   // BEGIN Non-volatile memory access
 2942 
 2943   // This encoding class is generated automatically from ad_encode.m4.
 2944   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2945   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2946     Register dst_reg = as_Register($dst$$reg);
 2947     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2948                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2949   %}
 2950 
 2951   // This encoding class is generated automatically from ad_encode.m4.
 2952   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2953   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2954     Register dst_reg = as_Register($dst$$reg);
 2955     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2956                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2957   %}
 2958 
 2959   // This encoding class is generated automatically from ad_encode.m4.
 2960   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2961   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2962     Register dst_reg = as_Register($dst$$reg);
 2963     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2964                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2965   %}
 2966 
 2967   // This encoding class is generated automatically from ad_encode.m4.
 2968   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2969   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2970     Register dst_reg = as_Register($dst$$reg);
 2971     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2972                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2973   %}
 2974 
 2975   // This encoding class is generated automatically from ad_encode.m4.
 2976   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2977   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2978     Register dst_reg = as_Register($dst$$reg);
 2979     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2980                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2981   %}
 2982 
 2983   // This encoding class is generated automatically from ad_encode.m4.
 2984   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2985   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2986     Register dst_reg = as_Register($dst$$reg);
 2987     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2988                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2989   %}
 2990 
 2991   // This encoding class is generated automatically from ad_encode.m4.
 2992   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2993   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2994     Register dst_reg = as_Register($dst$$reg);
 2995     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2997   %}
 2998 
 2999   // This encoding class is generated automatically from ad_encode.m4.
 3000   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3001   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3002     Register dst_reg = as_Register($dst$$reg);
 3003     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3004                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3005   %}
 3006 
 3007   // This encoding class is generated automatically from ad_encode.m4.
 3008   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3009   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3010     Register dst_reg = as_Register($dst$$reg);
 3011     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3012                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3013   %}
 3014 
 3015   // This encoding class is generated automatically from ad_encode.m4.
 3016   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3017   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3018     Register dst_reg = as_Register($dst$$reg);
 3019     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3020                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3021   %}
 3022 
 3023   // This encoding class is generated automatically from ad_encode.m4.
 3024   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3025   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3026     Register dst_reg = as_Register($dst$$reg);
 3027     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3028                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3029   %}
 3030 
 3031   // This encoding class is generated automatically from ad_encode.m4.
 3032   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3033   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3034     Register dst_reg = as_Register($dst$$reg);
 3035     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3036                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3037   %}
 3038 
 3039   // This encoding class is generated automatically from ad_encode.m4.
 3040   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3041   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3042     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3043     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3044                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3045   %}
 3046 
 3047   // This encoding class is generated automatically from ad_encode.m4.
 3048   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3049   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3050     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3051     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3052                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3053   %}
 3054 
 3055   // This encoding class is generated automatically from ad_encode.m4.
 3056   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3057   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3058     Register src_reg = as_Register($src$$reg);
 3059     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3060                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3061   %}
 3062 
 3063   // This encoding class is generated automatically from ad_encode.m4.
 3064   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3065   enc_class aarch64_enc_strb0(memory1 mem) %{
 3066     C2_MacroAssembler _masm(&cbuf);
 3067     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3068                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3069   %}
 3070 
 3071   // This encoding class is generated automatically from ad_encode.m4.
 3072   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3073   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3074     Register src_reg = as_Register($src$$reg);
 3075     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3076                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3077   %}
 3078 
 3079   // This encoding class is generated automatically from ad_encode.m4.
 3080   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3081   enc_class aarch64_enc_strh0(memory2 mem) %{
 3082     C2_MacroAssembler _masm(&cbuf);
 3083     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3084                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3085   %}
 3086 
 3087   // This encoding class is generated automatically from ad_encode.m4.
 3088   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3089   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3090     Register src_reg = as_Register($src$$reg);
 3091     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3092                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3093   %}
 3094 
 3095   // This encoding class is generated automatically from ad_encode.m4.
 3096   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3097   enc_class aarch64_enc_strw0(memory4 mem) %{
 3098     C2_MacroAssembler _masm(&cbuf);
 3099     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3100                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3101   %}
 3102 
 3103   // This encoding class is generated automatically from ad_encode.m4.
 3104   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3105   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3106     Register src_reg = as_Register($src$$reg);
 3107     // we sometimes get asked to store the stack pointer into the
 3108     // current thread -- we cannot do that directly on AArch64
 3109     if (src_reg == r31_sp) {
 3110       C2_MacroAssembler _masm(&cbuf);
 3111       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3112       __ mov(rscratch2, sp);
 3113       src_reg = rscratch2;
 3114     }
 3115     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3116                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3117   %}
 3118 
 3119   // This encoding class is generated automatically from ad_encode.m4.
 3120   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3121   enc_class aarch64_enc_str0(memory8 mem) %{
 3122     C2_MacroAssembler _masm(&cbuf);
 3123     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3124                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3125   %}
 3126 
 3127   // This encoding class is generated automatically from ad_encode.m4.
 3128   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3129   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3130     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3131     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3132                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3133   %}
 3134 
 3135   // This encoding class is generated automatically from ad_encode.m4.
 3136   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3137   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3138     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3139     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3140                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3141   %}
 3142 
 3143   // This encoding class is generated automatically from ad_encode.m4.
 3144   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3145   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3146       C2_MacroAssembler _masm(&cbuf);
 3147       __ membar(Assembler::StoreStore);
 3148       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3149                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3150   %}
 3151 
 3152   // END Non-volatile memory access
 3153 
 3154   // Vector loads and stores
 3155   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3156     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3157     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3158        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3159   %}
 3160 
 3161   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3162     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3163     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3164        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3165   %}
 3166 
 3167   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3168     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3169     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3170        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3171   %}
 3172 
 3173   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3174     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3175     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3176        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3177   %}
 3178 
 3179   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3180     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3181     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3182        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3183   %}
 3184 
 3185   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3186     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3187     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3188        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3189   %}
 3190 
 3191   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3192     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3193     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3194        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3195   %}
 3196 
 3197   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3198     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3199     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3200        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3201   %}
 3202 
 3203   // volatile loads and stores
 3204 
 3205   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3206     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3207                  rscratch1, stlrb);
 3208   %}
 3209 
 3210   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3211     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212                  rscratch1, stlrh);
 3213   %}
 3214 
 3215   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3216     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217                  rscratch1, stlrw);
 3218   %}
 3219 
 3220 
 3221   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3222     Register dst_reg = as_Register($dst$$reg);
 3223     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3224              rscratch1, ldarb);
 3225     __ sxtbw(dst_reg, dst_reg);
 3226   %}
 3227 
 3228   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3229     Register dst_reg = as_Register($dst$$reg);
 3230     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3231              rscratch1, ldarb);
 3232     __ sxtb(dst_reg, dst_reg);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3236     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237              rscratch1, ldarb);
 3238   %}
 3239 
 3240   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3241     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3242              rscratch1, ldarb);
 3243   %}
 3244 
 3245   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3246     Register dst_reg = as_Register($dst$$reg);
 3247     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248              rscratch1, ldarh);
 3249     __ sxthw(dst_reg, dst_reg);
 3250   %}
 3251 
 3252   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3253     Register dst_reg = as_Register($dst$$reg);
 3254     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3255              rscratch1, ldarh);
 3256     __ sxth(dst_reg, dst_reg);
 3257   %}
 3258 
 3259   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3260     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3261              rscratch1, ldarh);
 3262   %}
 3263 
 3264   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3265     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3266              rscratch1, ldarh);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3270     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3271              rscratch1, ldarw);
 3272   %}
 3273 
 3274   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3275     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3276              rscratch1, ldarw);
 3277   %}
 3278 
 3279   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3280     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3281              rscratch1, ldar);
 3282   %}
 3283 
 3284   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3285     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3286              rscratch1, ldarw);
 3287     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3291     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3292              rscratch1, ldar);
 3293     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3294   %}
 3295 
 3296   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3297     Register src_reg = as_Register($src$$reg);
 3298     // we sometimes get asked to store the stack pointer into the
 3299     // current thread -- we cannot do that directly on AArch64
 3300     if (src_reg == r31_sp) {
 3301       C2_MacroAssembler _masm(&cbuf);
 3302       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3303       __ mov(rscratch2, sp);
 3304       src_reg = rscratch2;
 3305     }
 3306     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3307                  rscratch1, stlr);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3311     {
 3312       C2_MacroAssembler _masm(&cbuf);
 3313       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3314       __ fmovs(rscratch2, src_reg);
 3315     }
 3316     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3317                  rscratch1, stlrw);
 3318   %}
 3319 
 3320   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3321     {
 3322       C2_MacroAssembler _masm(&cbuf);
 3323       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3324       __ fmovd(rscratch2, src_reg);
 3325     }
 3326     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3327                  rscratch1, stlr);
 3328   %}
 3329 
 3330   // synchronized read/update encodings
 3331 
 3332   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3333     C2_MacroAssembler _masm(&cbuf);
 3334     Register dst_reg = as_Register($dst$$reg);
 3335     Register base = as_Register($mem$$base);
 3336     int index = $mem$$index;
 3337     int scale = $mem$$scale;
 3338     int disp = $mem$$disp;
 3339     if (index == -1) {
 3340        if (disp != 0) {
 3341         __ lea(rscratch1, Address(base, disp));
 3342         __ ldaxr(dst_reg, rscratch1);
 3343       } else {
 3344         // TODO
 3345         // should we ever get anything other than this case?
 3346         __ ldaxr(dst_reg, base);
 3347       }
 3348     } else {
 3349       Register index_reg = as_Register(index);
 3350       if (disp == 0) {
 3351         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3352         __ ldaxr(dst_reg, rscratch1);
 3353       } else {
 3354         __ lea(rscratch1, Address(base, disp));
 3355         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3356         __ ldaxr(dst_reg, rscratch1);
 3357       }
 3358     }
 3359   %}
 3360 
 3361   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3362     C2_MacroAssembler _masm(&cbuf);
 3363     Register src_reg = as_Register($src$$reg);
 3364     Register base = as_Register($mem$$base);
 3365     int index = $mem$$index;
 3366     int scale = $mem$$scale;
 3367     int disp = $mem$$disp;
 3368     if (index == -1) {
 3369        if (disp != 0) {
 3370         __ lea(rscratch2, Address(base, disp));
 3371         __ stlxr(rscratch1, src_reg, rscratch2);
 3372       } else {
 3373         // TODO
 3374         // should we ever get anything other than this case?
 3375         __ stlxr(rscratch1, src_reg, base);
 3376       }
 3377     } else {
 3378       Register index_reg = as_Register(index);
 3379       if (disp == 0) {
 3380         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3381         __ stlxr(rscratch1, src_reg, rscratch2);
 3382       } else {
 3383         __ lea(rscratch2, Address(base, disp));
 3384         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3385         __ stlxr(rscratch1, src_reg, rscratch2);
 3386       }
 3387     }
 3388     __ cmpw(rscratch1, zr);
 3389   %}
 3390 
 3391   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3392     C2_MacroAssembler _masm(&cbuf);
 3393     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3394     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3395                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3396                /*weak*/ false, noreg);
 3397   %}
 3398 
 3399   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3400     C2_MacroAssembler _masm(&cbuf);
 3401     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3402     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3403                Assembler::word, /*acquire*/ false, /*release*/ true,
 3404                /*weak*/ false, noreg);
 3405   %}
 3406 
 3407   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3408     C2_MacroAssembler _masm(&cbuf);
 3409     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3410     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3411                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3412                /*weak*/ false, noreg);
 3413   %}
 3414 
 3415   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3416     C2_MacroAssembler _masm(&cbuf);
 3417     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3418     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3419                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3420                /*weak*/ false, noreg);
 3421   %}
 3422 
 3423 
 3424   // The only difference between aarch64_enc_cmpxchg and
 3425   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3426   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3427   // lock.
 3428   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3429     C2_MacroAssembler _masm(&cbuf);
 3430     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3431     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3432                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3433                /*weak*/ false, noreg);
 3434   %}
 3435 
 3436   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3437     C2_MacroAssembler _masm(&cbuf);
 3438     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3439     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3440                Assembler::word, /*acquire*/ true, /*release*/ true,
 3441                /*weak*/ false, noreg);
 3442   %}
 3443 
 3444   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3445     C2_MacroAssembler _masm(&cbuf);
 3446     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3447     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3448                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3449                /*weak*/ false, noreg);
 3450   %}
 3451 
 3452   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3453     C2_MacroAssembler _masm(&cbuf);
 3454     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3455     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3456                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3457                /*weak*/ false, noreg);
 3458   %}
 3459 
 3460   // auxiliary used for CompareAndSwapX to set result register
 3461   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register res_reg = as_Register($res$$reg);
 3464     __ cset(res_reg, Assembler::EQ);
 3465   %}
 3466 
 3467   // prefetch encodings
 3468 
 3469   enc_class aarch64_enc_prefetchw(memory mem) %{
 3470     C2_MacroAssembler _masm(&cbuf);
 3471     Register base = as_Register($mem$$base);
 3472     int index = $mem$$index;
 3473     int scale = $mem$$scale;
 3474     int disp = $mem$$disp;
 3475     if (index == -1) {
 3476       __ prfm(Address(base, disp), PSTL1KEEP);
 3477     } else {
 3478       Register index_reg = as_Register(index);
 3479       if (disp == 0) {
 3480         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3481       } else {
 3482         __ lea(rscratch1, Address(base, disp));
 3483 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3484       }
 3485     }
 3486   %}
 3487 
 3488   /// mov envcodings
 3489 
 3490   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3491     C2_MacroAssembler _masm(&cbuf);
 3492     uint32_t con = (uint32_t)$src$$constant;
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     if (con == 0) {
 3495       __ movw(dst_reg, zr);
 3496     } else {
 3497       __ movw(dst_reg, con);
 3498     }
 3499   %}
 3500 
 3501   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3502     C2_MacroAssembler _masm(&cbuf);
 3503     Register dst_reg = as_Register($dst$$reg);
 3504     uint64_t con = (uint64_t)$src$$constant;
 3505     if (con == 0) {
 3506       __ mov(dst_reg, zr);
 3507     } else {
 3508       __ mov(dst_reg, con);
 3509     }
 3510   %}
 3511 
 3512   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3513     C2_MacroAssembler _masm(&cbuf);
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     address con = (address)$src$$constant;
 3516     if (con == NULL || con == (address)1) {
 3517       ShouldNotReachHere();
 3518     } else {
 3519       relocInfo::relocType rtype = $src->constant_reloc();
 3520       if (rtype == relocInfo::oop_type) {
 3521         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3522       } else if (rtype == relocInfo::metadata_type) {
 3523         __ mov_metadata(dst_reg, (Metadata*)con);
 3524       } else {
 3525         assert(rtype == relocInfo::none, "unexpected reloc type");
 3526         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3527           __ mov(dst_reg, con);
 3528         } else {
 3529           uint64_t offset;
 3530           __ adrp(dst_reg, con, offset);
 3531           __ add(dst_reg, dst_reg, offset);
 3532         }
 3533       }
 3534     }
 3535   %}
 3536 
 3537   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3538     C2_MacroAssembler _masm(&cbuf);
 3539     Register dst_reg = as_Register($dst$$reg);
 3540     __ mov(dst_reg, zr);
 3541   %}
 3542 
 3543   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3544     C2_MacroAssembler _masm(&cbuf);
 3545     Register dst_reg = as_Register($dst$$reg);
 3546     __ mov(dst_reg, (uint64_t)1);
 3547   %}
 3548 
 3549   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3550     C2_MacroAssembler _masm(&cbuf);
 3551     __ load_byte_map_base($dst$$Register);
 3552   %}
 3553 
 3554   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3555     C2_MacroAssembler _masm(&cbuf);
 3556     Register dst_reg = as_Register($dst$$reg);
 3557     address con = (address)$src$$constant;
 3558     if (con == NULL) {
 3559       ShouldNotReachHere();
 3560     } else {
 3561       relocInfo::relocType rtype = $src->constant_reloc();
 3562       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3563       __ set_narrow_oop(dst_reg, (jobject)con);
 3564     }
 3565   %}
 3566 
 3567   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3568     C2_MacroAssembler _masm(&cbuf);
 3569     Register dst_reg = as_Register($dst$$reg);
 3570     __ mov(dst_reg, zr);
 3571   %}
 3572 
 3573   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3574     C2_MacroAssembler _masm(&cbuf);
 3575     Register dst_reg = as_Register($dst$$reg);
 3576     address con = (address)$src$$constant;
 3577     if (con == NULL) {
 3578       ShouldNotReachHere();
 3579     } else {
 3580       relocInfo::relocType rtype = $src->constant_reloc();
 3581       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3582       __ set_narrow_klass(dst_reg, (Klass *)con);
 3583     }
 3584   %}
 3585 
 3586   // arithmetic encodings
 3587 
 3588   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register dst_reg = as_Register($dst$$reg);
 3591     Register src_reg = as_Register($src1$$reg);
 3592     int32_t con = (int32_t)$src2$$constant;
 3593     // add has primary == 0, subtract has primary == 1
 3594     if ($primary) { con = -con; }
 3595     if (con < 0) {
 3596       __ subw(dst_reg, src_reg, -con);
 3597     } else {
 3598       __ addw(dst_reg, src_reg, con);
 3599     }
 3600   %}
 3601 
 3602   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3603     C2_MacroAssembler _masm(&cbuf);
 3604     Register dst_reg = as_Register($dst$$reg);
 3605     Register src_reg = as_Register($src1$$reg);
 3606     int32_t con = (int32_t)$src2$$constant;
 3607     // add has primary == 0, subtract has primary == 1
 3608     if ($primary) { con = -con; }
 3609     if (con < 0) {
 3610       __ sub(dst_reg, src_reg, -con);
 3611     } else {
 3612       __ add(dst_reg, src_reg, con);
 3613     }
 3614   %}
 3615 
 3616   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3617     C2_MacroAssembler _masm(&cbuf);
 3618    Register dst_reg = as_Register($dst$$reg);
 3619    Register src1_reg = as_Register($src1$$reg);
 3620    Register src2_reg = as_Register($src2$$reg);
 3621     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3625     C2_MacroAssembler _masm(&cbuf);
 3626    Register dst_reg = as_Register($dst$$reg);
 3627    Register src1_reg = as_Register($src1$$reg);
 3628    Register src2_reg = as_Register($src2$$reg);
 3629     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3630   %}
 3631 
 3632   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3633     C2_MacroAssembler _masm(&cbuf);
 3634    Register dst_reg = as_Register($dst$$reg);
 3635    Register src1_reg = as_Register($src1$$reg);
 3636    Register src2_reg = as_Register($src2$$reg);
 3637     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3638   %}
 3639 
 3640   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3641     C2_MacroAssembler _masm(&cbuf);
 3642    Register dst_reg = as_Register($dst$$reg);
 3643    Register src1_reg = as_Register($src1$$reg);
 3644    Register src2_reg = as_Register($src2$$reg);
 3645     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3646   %}
 3647 
 3648   // compare instruction encodings
 3649 
 3650   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652     Register reg1 = as_Register($src1$$reg);
 3653     Register reg2 = as_Register($src2$$reg);
 3654     __ cmpw(reg1, reg2);
 3655   %}
 3656 
 3657   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3658     C2_MacroAssembler _masm(&cbuf);
 3659     Register reg = as_Register($src1$$reg);
 3660     int32_t val = $src2$$constant;
 3661     if (val >= 0) {
 3662       __ subsw(zr, reg, val);
 3663     } else {
 3664       __ addsw(zr, reg, -val);
 3665     }
 3666   %}
 3667 
 3668   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3669     C2_MacroAssembler _masm(&cbuf);
 3670     Register reg1 = as_Register($src1$$reg);
 3671     uint32_t val = (uint32_t)$src2$$constant;
 3672     __ movw(rscratch1, val);
 3673     __ cmpw(reg1, rscratch1);
 3674   %}
 3675 
 3676   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3677     C2_MacroAssembler _masm(&cbuf);
 3678     Register reg1 = as_Register($src1$$reg);
 3679     Register reg2 = as_Register($src2$$reg);
 3680     __ cmp(reg1, reg2);
 3681   %}
 3682 
 3683   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3684     C2_MacroAssembler _masm(&cbuf);
 3685     Register reg = as_Register($src1$$reg);
 3686     int64_t val = $src2$$constant;
 3687     if (val >= 0) {
 3688       __ subs(zr, reg, val);
 3689     } else if (val != -val) {
 3690       __ adds(zr, reg, -val);
 3691     } else {
 3692     // aargh, Long.MIN_VALUE is a special case
 3693       __ orr(rscratch1, zr, (uint64_t)val);
 3694       __ subs(zr, reg, rscratch1);
 3695     }
 3696   %}
 3697 
 3698   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3699     C2_MacroAssembler _masm(&cbuf);
 3700     Register reg1 = as_Register($src1$$reg);
 3701     uint64_t val = (uint64_t)$src2$$constant;
 3702     __ mov(rscratch1, val);
 3703     __ cmp(reg1, rscratch1);
 3704   %}
 3705 
 3706   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3707     C2_MacroAssembler _masm(&cbuf);
 3708     Register reg1 = as_Register($src1$$reg);
 3709     Register reg2 = as_Register($src2$$reg);
 3710     __ cmp(reg1, reg2);
 3711   %}
 3712 
 3713   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3714     C2_MacroAssembler _masm(&cbuf);
 3715     Register reg1 = as_Register($src1$$reg);
 3716     Register reg2 = as_Register($src2$$reg);
 3717     __ cmpw(reg1, reg2);
 3718   %}
 3719 
 3720   enc_class aarch64_enc_testp(iRegP src) %{
 3721     C2_MacroAssembler _masm(&cbuf);
 3722     Register reg = as_Register($src$$reg);
 3723     __ cmp(reg, zr);
 3724   %}
 3725 
 3726   enc_class aarch64_enc_testn(iRegN src) %{
 3727     C2_MacroAssembler _masm(&cbuf);
 3728     Register reg = as_Register($src$$reg);
 3729     __ cmpw(reg, zr);
 3730   %}
 3731 
 3732   enc_class aarch64_enc_b(label lbl) %{
 3733     C2_MacroAssembler _masm(&cbuf);
 3734     Label *L = $lbl$$label;
 3735     __ b(*L);
 3736   %}
 3737 
 3738   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3739     C2_MacroAssembler _masm(&cbuf);
 3740     Label *L = $lbl$$label;
 3741     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3742   %}
 3743 
 3744   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3745     C2_MacroAssembler _masm(&cbuf);
 3746     Label *L = $lbl$$label;
 3747     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3748   %}
 3749 
 3750   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3751   %{
 3752      Register sub_reg = as_Register($sub$$reg);
 3753      Register super_reg = as_Register($super$$reg);
 3754      Register temp_reg = as_Register($temp$$reg);
 3755      Register result_reg = as_Register($result$$reg);
 3756 
 3757      Label miss;
 3758      C2_MacroAssembler _masm(&cbuf);
 3759      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3760                                      NULL, &miss,
 3761                                      /*set_cond_codes:*/ true);
 3762      if ($primary) {
 3763        __ mov(result_reg, zr);
 3764      }
 3765      __ bind(miss);
 3766   %}
 3767 
 3768   enc_class aarch64_enc_java_static_call(method meth) %{
 3769     C2_MacroAssembler _masm(&cbuf);
 3770 
 3771     address addr = (address)$meth$$method;
 3772     address call;
 3773     if (!_method) {
 3774       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3775       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3776       if (call == NULL) {
 3777         ciEnv::current()->record_failure("CodeCache is full");
 3778         return;
 3779       }
 3780     } else {
 3781       int method_index = resolved_method_index(cbuf);
 3782       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3783                                                   : static_call_Relocation::spec(method_index);
 3784       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3785       if (call == NULL) {
 3786         ciEnv::current()->record_failure("CodeCache is full");
 3787         return;
 3788       }
 3789       // Emit stub for static call
 3790       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3791       if (stub == NULL) {
 3792         ciEnv::current()->record_failure("CodeCache is full");
 3793         return;
 3794       }
 3795     }
 3796 
 3797     __ post_call_nop();
 3798 
 3799     // Only non uncommon_trap calls need to reinitialize ptrue.
 3800     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3801       __ reinitialize_ptrue();
 3802     }
 3803   %}
 3804 
 3805   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3806     C2_MacroAssembler _masm(&cbuf);
 3807     int method_index = resolved_method_index(cbuf);
 3808     address call = __ ic_call((address)$meth$$method, method_index);
 3809     __ post_call_nop();
 3810     if (call == NULL) {
 3811       ciEnv::current()->record_failure("CodeCache is full");
 3812       return;
 3813     } else if (Compile::current()->max_vector_size() > 0) {
 3814       __ reinitialize_ptrue();
 3815     }
 3816   %}
 3817 
 3818   enc_class aarch64_enc_call_epilog() %{
 3819     C2_MacroAssembler _masm(&cbuf);
 3820     if (VerifyStackAtCalls) {
 3821       // Check that stack depth is unchanged: find majik cookie on stack
 3822       __ call_Unimplemented();
 3823     }
 3824   %}
 3825 
 3826   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3827     C2_MacroAssembler _masm(&cbuf);
 3828 
 3829     // some calls to generated routines (arraycopy code) are scheduled
 3830     // by C2 as runtime calls. if so we can call them using a br (they
 3831     // will be in a reachable segment) otherwise we have to use a blr
 3832     // which loads the absolute address into a register.
 3833     address entry = (address)$meth$$method;
 3834     CodeBlob *cb = CodeCache::find_blob(entry);
 3835     if (cb) {
 3836       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3837       __ post_call_nop();
 3838       if (call == NULL) {
 3839         ciEnv::current()->record_failure("CodeCache is full");
 3840         return;
 3841       }
 3842     } else {
 3843       Label retaddr;
 3844       __ adr(rscratch2, retaddr);
 3845       __ lea(rscratch1, RuntimeAddress(entry));
 3846       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3847       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3848       __ blr(rscratch1);
 3849       __ bind(retaddr);
 3850       // __ post_call_nop(); -- can we have one here? do we need one?
 3851       __ add(sp, sp, 2 * wordSize);
 3852     }
 3853     if (Compile::current()->max_vector_size() > 0) {
 3854       __ reinitialize_ptrue();
 3855     }
 3856   %}
 3857 
 3858   enc_class aarch64_enc_rethrow() %{
 3859     C2_MacroAssembler _masm(&cbuf);
 3860     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3861   %}
 3862 
 3863   enc_class aarch64_enc_ret() %{
 3864     C2_MacroAssembler _masm(&cbuf);
 3865 #ifdef ASSERT
 3866     if (Compile::current()->max_vector_size() > 0) {
 3867       __ verify_ptrue();
 3868     }
 3869 #endif
 3870     __ ret(lr);
 3871   %}
 3872 
 3873   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3874     C2_MacroAssembler _masm(&cbuf);
 3875     Register target_reg = as_Register($jump_target$$reg);
 3876     __ br(target_reg);
 3877   %}
 3878 
 3879   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3880     C2_MacroAssembler _masm(&cbuf);
 3881     Register target_reg = as_Register($jump_target$$reg);
 3882     // exception oop should be in r0
 3883     // ret addr has been popped into lr
 3884     // callee expects it in r3
 3885     __ mov(r3, lr);
 3886     __ br(target_reg);
 3887   %}
 3888 
 3889   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3890     C2_MacroAssembler _masm(&cbuf);
 3891     Register oop = as_Register($object$$reg);
 3892     Register box = as_Register($box$$reg);
 3893     Register disp_hdr = as_Register($tmp$$reg);
 3894     Register tmp = as_Register($tmp2$$reg);
 3895     Label cont;
 3896     Label object_has_monitor;
 3897     Label cas_failed;
 3898 
 3899     assert_different_registers(oop, box, tmp, disp_hdr);
 3900 
 3901     // Load markWord from object into displaced_header.
 3902     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3903 
 3904     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3905       __ load_klass(tmp, oop);
 3906       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3907       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3908       __ br(Assembler::NE, cont);
 3909     }
 3910 
 3911     // Check for existing monitor
 3912     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3913 
 3914     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3915     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3916 
 3917     // Initialize the box. (Must happen before we update the object mark!)
 3918     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3919 
 3920     // Compare object markWord with an unlocked value (tmp) and if
 3921     // equal exchange the stack address of our box with object markWord.
 3922     // On failure disp_hdr contains the possibly locked markWord.
 3923     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3924                /*release*/ true, /*weak*/ false, disp_hdr);
 3925     __ br(Assembler::EQ, cont);
 3926 
 3927     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3928 
 3929     // If the compare-and-exchange succeeded, then we found an unlocked
 3930     // object, will have now locked it will continue at label cont
 3931 
 3932     __ bind(cas_failed);
 3933     // We did not see an unlocked object so try the fast recursive case.
 3934 
 3935     // Check if the owner is self by comparing the value in the
 3936     // markWord of object (disp_hdr) with the stack pointer.
 3937     __ mov(rscratch1, sp);
 3938     __ sub(disp_hdr, disp_hdr, rscratch1);
 3939     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3940     // If condition is true we are cont and hence we can store 0 as the
 3941     // displaced header in the box, which indicates that it is a recursive lock.
 3942     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3943     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3944 
 3945     __ b(cont);
 3946 
 3947     // Handle existing monitor.
 3948     __ bind(object_has_monitor);
 3949 
 3950     // The object's monitor m is unlocked iff m->owner == NULL,
 3951     // otherwise m->owner may contain a thread or a stack address.
 3952     //
 3953     // Try to CAS m->owner from NULL to current thread.
 3954     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3955     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3956                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3957 
 3958     // Store a non-null value into the box to avoid looking like a re-entrant
 3959     // lock. The fast-path monitor unlock code checks for
 3960     // markWord::monitor_value so use markWord::unused_mark which has the
 3961     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3962     __ mov(tmp, (address)markWord::unused_mark().value());
 3963     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3964 
 3965     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3966 
 3967     __ cmp(rscratch1, rthread);
 3968     __ br(Assembler::NE, cont); // Check for recursive locking
 3969 
 3970     // Recursive lock case
 3971     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3972     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3973 
 3974     __ bind(cont);
 3975     // flag == EQ indicates success
 3976     // flag == NE indicates failure
 3977   %}
 3978 
 3979   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3980     C2_MacroAssembler _masm(&cbuf);
 3981     Register oop = as_Register($object$$reg);
 3982     Register box = as_Register($box$$reg);
 3983     Register disp_hdr = as_Register($tmp$$reg);
 3984     Register tmp = as_Register($tmp2$$reg);
 3985     Label cont;
 3986     Label object_has_monitor;
 3987 
 3988     assert_different_registers(oop, box, tmp, disp_hdr);
 3989 
 3990     // Find the lock address and load the displaced header from the stack.
 3991     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3992 
 3993     // If the displaced header is 0, we have a recursive unlock.
 3994     __ cmp(disp_hdr, zr);
 3995     __ br(Assembler::EQ, cont);
 3996 
 3997     // Handle existing monitor.
 3998     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3999     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 4000 
 4001     // Check if it is still a light weight lock, this is is true if we
 4002     // see the stack address of the basicLock in the markWord of the
 4003     // object.
 4004 
 4005     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4006                /*release*/ true, /*weak*/ false, tmp);
 4007     __ b(cont);
 4008 
 4009     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4010 
 4011     // Handle existing monitor.
 4012     __ bind(object_has_monitor);
 4013     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4014     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4015     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4016     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4017 
 4018     Label notRecursive;
 4019     __ cmp(rscratch1, rthread);
 4020     __ br(Assembler::NE, cont);
 4021 
 4022     __ cbz(disp_hdr, notRecursive);
 4023 
 4024     // Recursive lock
 4025     __ sub(disp_hdr, disp_hdr, 1u);
 4026     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4027     // flag == EQ was set in the ownership check above
 4028     __ b(cont);
 4029 
 4030     __ bind(notRecursive);
 4031     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4032     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4033     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4034     __ cmp(rscratch1, zr); // Sets flags for result
 4035     __ cbnz(rscratch1, cont);
 4036     // need a release store here
 4037     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4038     __ stlr(zr, tmp); // set unowned
 4039 
 4040     __ bind(cont);
 4041     // flag == EQ indicates success
 4042     // flag == NE indicates failure
 4043   %}
 4044 
 4045 %}
 4046 
 4047 //----------FRAME--------------------------------------------------------------
 4048 // Definition of frame structure and management information.
 4049 //
 4050 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4051 //                             |   (to get allocators register number
 4052 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4053 //  r   CALLER     |        |
 4054 //  o     |        +--------+      pad to even-align allocators stack-slot
 4055 //  w     V        |  pad0  |        numbers; owned by CALLER
 4056 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4057 //  h     ^        |   in   |  5
 4058 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4059 //  |     |        |        |  3
 4060 //  |     |        +--------+
 4061 //  V     |        | old out|      Empty on Intel, window on Sparc
 4062 //        |    old |preserve|      Must be even aligned.
 4063 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4064 //        |        |   in   |  3   area for Intel ret address
 4065 //     Owned by    |preserve|      Empty on Sparc.
 4066 //       SELF      +--------+
 4067 //        |        |  pad2  |  2   pad to align old SP
 4068 //        |        +--------+  1
 4069 //        |        | locks  |  0
 4070 //        |        +--------+----> OptoReg::stack0(), even aligned
 4071 //        |        |  pad1  | 11   pad to align new SP
 4072 //        |        +--------+
 4073 //        |        |        | 10
 4074 //        |        | spills |  9   spills
 4075 //        V        |        |  8   (pad0 slot for callee)
 4076 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4077 //        ^        |  out   |  7
 4078 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4079 //     Owned by    +--------+
 4080 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4081 //        |    new |preserve|      Must be even-aligned.
 4082 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4083 //        |        |        |
 4084 //
 4085 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4086 //         known from SELF's arguments and the Java calling convention.
 4087 //         Region 6-7 is determined per call site.
 4088 // Note 2: If the calling convention leaves holes in the incoming argument
 4089 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4090 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4091 //         incoming area, as the Java calling convention is completely under
 4092 //         the control of the AD file.  Doubles can be sorted and packed to
 4093 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4094 //         varargs C calling conventions.
 4095 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4096 //         even aligned with pad0 as needed.
 4097 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4098 //           (the latter is true on Intel but is it false on AArch64?)
 4099 //         region 6-11 is even aligned; it may be padded out more so that
 4100 //         the region from SP to FP meets the minimum stack alignment.
 4101 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4102 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4103 //         SP meets the minimum alignment.
 4104 
 4105 frame %{
 4106   // These three registers define part of the calling convention
 4107   // between compiled code and the interpreter.
 4108 
 4109   // Inline Cache Register or Method for I2C.
 4110   inline_cache_reg(R12);
 4111 
 4112   // Number of stack slots consumed by locking an object
 4113   sync_stack_slots(2);
 4114 
 4115   // Compiled code's Frame Pointer
 4116   frame_pointer(R31);
 4117 
 4118   // Interpreter stores its frame pointer in a register which is
 4119   // stored to the stack by I2CAdaptors.
 4120   // I2CAdaptors convert from interpreted java to compiled java.
 4121   interpreter_frame_pointer(R29);
 4122 
 4123   // Stack alignment requirement
 4124   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4125 
 4126   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4127   // for calls to C.  Supports the var-args backing area for register parms.
 4128   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4129 
 4130   // The after-PROLOG location of the return address.  Location of
 4131   // return address specifies a type (REG or STACK) and a number
 4132   // representing the register number (i.e. - use a register name) or
 4133   // stack slot.
 4134   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4135   // Otherwise, it is above the locks and verification slot and alignment word
 4136   // TODO this may well be correct but need to check why that - 2 is there
 4137   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4138   // which folds in the space used for monitors
 4139   return_addr(STACK - 2 +
 4140               align_up((Compile::current()->in_preserve_stack_slots() +
 4141                         Compile::current()->fixed_slots()),
 4142                        stack_alignment_in_slots()));
 4143 
 4144   // Location of compiled Java return values.  Same as C for now.
 4145   return_value
 4146   %{
 4147     // TODO do we allow ideal_reg == Op_RegN???
 4148     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4149            "only return normal values");
 4150 
 4151     static const int lo[Op_RegL + 1] = { // enum name
 4152       0,                                 // Op_Node
 4153       0,                                 // Op_Set
 4154       R0_num,                            // Op_RegN
 4155       R0_num,                            // Op_RegI
 4156       R0_num,                            // Op_RegP
 4157       V0_num,                            // Op_RegF
 4158       V0_num,                            // Op_RegD
 4159       R0_num                             // Op_RegL
 4160     };
 4161 
 4162     static const int hi[Op_RegL + 1] = { // enum name
 4163       0,                                 // Op_Node
 4164       0,                                 // Op_Set
 4165       OptoReg::Bad,                      // Op_RegN
 4166       OptoReg::Bad,                      // Op_RegI
 4167       R0_H_num,                          // Op_RegP
 4168       OptoReg::Bad,                      // Op_RegF
 4169       V0_H_num,                          // Op_RegD
 4170       R0_H_num                           // Op_RegL
 4171     };
 4172 
 4173     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4174   %}
 4175 %}
 4176 
 4177 //----------ATTRIBUTES---------------------------------------------------------
 4178 //----------Operand Attributes-------------------------------------------------
 4179 op_attrib op_cost(1);        // Required cost attribute
 4180 
 4181 //----------Instruction Attributes---------------------------------------------
 4182 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4183 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4184 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4185                                 // a non-matching short branch variant
 4186                                 // of some long branch?
 4187 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4188                                 // be a power of 2) specifies the
 4189                                 // alignment that some part of the
 4190                                 // instruction (not necessarily the
 4191                                 // start) requires.  If > 1, a
 4192                                 // compute_padding() function must be
 4193                                 // provided for the instruction
 4194 
 4195 //----------OPERANDS-----------------------------------------------------------
 4196 // Operand definitions must precede instruction definitions for correct parsing
 4197 // in the ADLC because operands constitute user defined types which are used in
 4198 // instruction definitions.
 4199 
 4200 //----------Simple Operands----------------------------------------------------
 4201 
 4202 // Integer operands 32 bit
 4203 // 32 bit immediate
 4204 operand immI()
 4205 %{
 4206   match(ConI);
 4207 
 4208   op_cost(0);
 4209   format %{ %}
 4210   interface(CONST_INTER);
 4211 %}
 4212 
 4213 // 32 bit zero
 4214 operand immI0()
 4215 %{
 4216   predicate(n->get_int() == 0);
 4217   match(ConI);
 4218 
 4219   op_cost(0);
 4220   format %{ %}
 4221   interface(CONST_INTER);
 4222 %}
 4223 
 4224 // 32 bit unit increment
 4225 operand immI_1()
 4226 %{
 4227   predicate(n->get_int() == 1);
 4228   match(ConI);
 4229 
 4230   op_cost(0);
 4231   format %{ %}
 4232   interface(CONST_INTER);
 4233 %}
 4234 
 4235 // 32 bit unit decrement
 4236 operand immI_M1()
 4237 %{
 4238   predicate(n->get_int() == -1);
 4239   match(ConI);
 4240 
 4241   op_cost(0);
 4242   format %{ %}
 4243   interface(CONST_INTER);
 4244 %}
 4245 
 4246 // Shift values for add/sub extension shift
 4247 operand immIExt()
 4248 %{
 4249   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4250   match(ConI);
 4251 
 4252   op_cost(0);
 4253   format %{ %}
 4254   interface(CONST_INTER);
 4255 %}
 4256 
 4257 operand immI_gt_1()
 4258 %{
 4259   predicate(n->get_int() > 1);
 4260   match(ConI);
 4261 
 4262   op_cost(0);
 4263   format %{ %}
 4264   interface(CONST_INTER);
 4265 %}
 4266 
 4267 operand immI_le_4()
 4268 %{
 4269   predicate(n->get_int() <= 4);
 4270   match(ConI);
 4271 
 4272   op_cost(0);
 4273   format %{ %}
 4274   interface(CONST_INTER);
 4275 %}
 4276 
 4277 operand immI_31()
 4278 %{
 4279   predicate(n->get_int() == 31);
 4280   match(ConI);
 4281 
 4282   op_cost(0);
 4283   format %{ %}
 4284   interface(CONST_INTER);
 4285 %}
 4286 
 4287 operand immI_2()
 4288 %{
 4289   predicate(n->get_int() == 2);
 4290   match(ConI);
 4291 
 4292   op_cost(0);
 4293   format %{ %}
 4294   interface(CONST_INTER);
 4295 %}
 4296 
 4297 operand immI_4()
 4298 %{
 4299   predicate(n->get_int() == 4);
 4300   match(ConI);
 4301 
 4302   op_cost(0);
 4303   format %{ %}
 4304   interface(CONST_INTER);
 4305 %}
 4306 
 4307 operand immI_8()
 4308 %{
 4309   predicate(n->get_int() == 8);
 4310   match(ConI);
 4311 
 4312   op_cost(0);
 4313   format %{ %}
 4314   interface(CONST_INTER);
 4315 %}
 4316 
 4317 operand immI_16()
 4318 %{
 4319   predicate(n->get_int() == 16);
 4320   match(ConI);
 4321 
 4322   op_cost(0);
 4323   format %{ %}
 4324   interface(CONST_INTER);
 4325 %}
 4326 
 4327 operand immI_24()
 4328 %{
 4329   predicate(n->get_int() == 24);
 4330   match(ConI);
 4331 
 4332   op_cost(0);
 4333   format %{ %}
 4334   interface(CONST_INTER);
 4335 %}
 4336 
 4337 operand immI_32()
 4338 %{
 4339   predicate(n->get_int() == 32);
 4340   match(ConI);
 4341 
 4342   op_cost(0);
 4343   format %{ %}
 4344   interface(CONST_INTER);
 4345 %}
 4346 
 4347 operand immI_48()
 4348 %{
 4349   predicate(n->get_int() == 48);
 4350   match(ConI);
 4351 
 4352   op_cost(0);
 4353   format %{ %}
 4354   interface(CONST_INTER);
 4355 %}
 4356 
 4357 operand immI_56()
 4358 %{
 4359   predicate(n->get_int() == 56);
 4360   match(ConI);
 4361 
 4362   op_cost(0);
 4363   format %{ %}
 4364   interface(CONST_INTER);
 4365 %}
 4366 
 4367 operand immI_63()
 4368 %{
 4369   predicate(n->get_int() == 63);
 4370   match(ConI);
 4371 
 4372   op_cost(0);
 4373   format %{ %}
 4374   interface(CONST_INTER);
 4375 %}
 4376 
 4377 operand immI_64()
 4378 %{
 4379   predicate(n->get_int() == 64);
 4380   match(ConI);
 4381 
 4382   op_cost(0);
 4383   format %{ %}
 4384   interface(CONST_INTER);
 4385 %}
 4386 
 4387 operand immI_255()
 4388 %{
 4389   predicate(n->get_int() == 255);
 4390   match(ConI);
 4391 
 4392   op_cost(0);
 4393   format %{ %}
 4394   interface(CONST_INTER);
 4395 %}
 4396 
 4397 operand immI_65535()
 4398 %{
 4399   predicate(n->get_int() == 65535);
 4400   match(ConI);
 4401 
 4402   op_cost(0);
 4403   format %{ %}
 4404   interface(CONST_INTER);
 4405 %}
 4406 
 4407 operand immL_255()
 4408 %{
 4409   predicate(n->get_long() == 255L);
 4410   match(ConL);
 4411 
 4412   op_cost(0);
 4413   format %{ %}
 4414   interface(CONST_INTER);
 4415 %}
 4416 
 4417 operand immL_65535()
 4418 %{
 4419   predicate(n->get_long() == 65535L);
 4420   match(ConL);
 4421 
 4422   op_cost(0);
 4423   format %{ %}
 4424   interface(CONST_INTER);
 4425 %}
 4426 
 4427 operand immL_4294967295()
 4428 %{
 4429   predicate(n->get_long() == 4294967295L);
 4430   match(ConL);
 4431 
 4432   op_cost(0);
 4433   format %{ %}
 4434   interface(CONST_INTER);
 4435 %}
 4436 
 4437 operand immL_bitmask()
 4438 %{
 4439   predicate((n->get_long() != 0)
 4440             && ((n->get_long() & 0xc000000000000000l) == 0)
 4441             && is_power_of_2(n->get_long() + 1));
 4442   match(ConL);
 4443 
 4444   op_cost(0);
 4445   format %{ %}
 4446   interface(CONST_INTER);
 4447 %}
 4448 
 4449 operand immI_bitmask()
 4450 %{
 4451   predicate((n->get_int() != 0)
 4452             && ((n->get_int() & 0xc0000000) == 0)
 4453             && is_power_of_2(n->get_int() + 1));
 4454   match(ConI);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 operand immL_positive_bitmaskI()
 4462 %{
 4463   predicate((n->get_long() != 0)
 4464             && ((julong)n->get_long() < 0x80000000ULL)
 4465             && is_power_of_2(n->get_long() + 1));
 4466   match(ConL);
 4467 
 4468   op_cost(0);
 4469   format %{ %}
 4470   interface(CONST_INTER);
 4471 %}
 4472 
 4473 // Scale values for scaled offset addressing modes (up to long but not quad)
 4474 operand immIScale()
 4475 %{
 4476   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4477   match(ConI);
 4478 
 4479   op_cost(0);
 4480   format %{ %}
 4481   interface(CONST_INTER);
 4482 %}
 4483 
 4484 // 26 bit signed offset -- for pc-relative branches
 4485 operand immI26()
 4486 %{
 4487   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4488   match(ConI);
 4489 
 4490   op_cost(0);
 4491   format %{ %}
 4492   interface(CONST_INTER);
 4493 %}
 4494 
 4495 // 19 bit signed offset -- for pc-relative loads
 4496 operand immI19()
 4497 %{
 4498   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4499   match(ConI);
 4500 
 4501   op_cost(0);
 4502   format %{ %}
 4503   interface(CONST_INTER);
 4504 %}
 4505 
 4506 // 12 bit unsigned offset -- for base plus immediate loads
 4507 operand immIU12()
 4508 %{
 4509   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4510   match(ConI);
 4511 
 4512   op_cost(0);
 4513   format %{ %}
 4514   interface(CONST_INTER);
 4515 %}
 4516 
 4517 operand immLU12()
 4518 %{
 4519   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4520   match(ConL);
 4521 
 4522   op_cost(0);
 4523   format %{ %}
 4524   interface(CONST_INTER);
 4525 %}
 4526 
 4527 // Offset for scaled or unscaled immediate loads and stores
 4528 operand immIOffset()
 4529 %{
 4530   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4531   match(ConI);
 4532 
 4533   op_cost(0);
 4534   format %{ %}
 4535   interface(CONST_INTER);
 4536 %}
 4537 
 4538 operand immIOffset1()
 4539 %{
 4540   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4541   match(ConI);
 4542 
 4543   op_cost(0);
 4544   format %{ %}
 4545   interface(CONST_INTER);
 4546 %}
 4547 
 4548 operand immIOffset2()
 4549 %{
 4550   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4551   match(ConI);
 4552 
 4553   op_cost(0);
 4554   format %{ %}
 4555   interface(CONST_INTER);
 4556 %}
 4557 
 4558 operand immIOffset4()
 4559 %{
 4560   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4561   match(ConI);
 4562 
 4563   op_cost(0);
 4564   format %{ %}
 4565   interface(CONST_INTER);
 4566 %}
 4567 
 4568 operand immIOffset8()
 4569 %{
 4570   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4571   match(ConI);
 4572 
 4573   op_cost(0);
 4574   format %{ %}
 4575   interface(CONST_INTER);
 4576 %}
 4577 
 4578 operand immIOffset16()
 4579 %{
 4580   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4581   match(ConI);
 4582 
 4583   op_cost(0);
 4584   format %{ %}
 4585   interface(CONST_INTER);
 4586 %}
 4587 
 4588 operand immLoffset()
 4589 %{
 4590   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4591   match(ConL);
 4592 
 4593   op_cost(0);
 4594   format %{ %}
 4595   interface(CONST_INTER);
 4596 %}
 4597 
 4598 operand immLoffset1()
 4599 %{
 4600   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4601   match(ConL);
 4602 
 4603   op_cost(0);
 4604   format %{ %}
 4605   interface(CONST_INTER);
 4606 %}
 4607 
 4608 operand immLoffset2()
 4609 %{
 4610   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4611   match(ConL);
 4612 
 4613   op_cost(0);
 4614   format %{ %}
 4615   interface(CONST_INTER);
 4616 %}
 4617 
 4618 operand immLoffset4()
 4619 %{
 4620   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4621   match(ConL);
 4622 
 4623   op_cost(0);
 4624   format %{ %}
 4625   interface(CONST_INTER);
 4626 %}
 4627 
 4628 operand immLoffset8()
 4629 %{
 4630   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4631   match(ConL);
 4632 
 4633   op_cost(0);
 4634   format %{ %}
 4635   interface(CONST_INTER);
 4636 %}
 4637 
 4638 operand immLoffset16()
 4639 %{
 4640   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4641   match(ConL);
 4642 
 4643   op_cost(0);
 4644   format %{ %}
 4645   interface(CONST_INTER);
 4646 %}
 4647 
 4648 // 8 bit signed value.
 4649 operand immI8()
 4650 %{
 4651   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4652   match(ConI);
 4653 
 4654   op_cost(0);
 4655   format %{ %}
 4656   interface(CONST_INTER);
 4657 %}
 4658 
 4659 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4660 operand immI8_shift8()
 4661 %{
 4662   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4663             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4664   match(ConI);
 4665 
 4666   op_cost(0);
 4667   format %{ %}
 4668   interface(CONST_INTER);
 4669 %}
 4670 
 4671 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4672 operand immL8_shift8()
 4673 %{
 4674   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4675             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4676   match(ConL);
 4677 
 4678   op_cost(0);
 4679   format %{ %}
 4680   interface(CONST_INTER);
 4681 %}
 4682 
 4683 // 8 bit integer valid for vector add sub immediate
 4684 operand immBAddSubV()
 4685 %{
 4686   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4687   match(ConI);
 4688 
 4689   op_cost(0);
 4690   format %{ %}
 4691   interface(CONST_INTER);
 4692 %}
 4693 
 4694 // 32 bit integer valid for add sub immediate
 4695 operand immIAddSub()
 4696 %{
 4697   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4698   match(ConI);
 4699   op_cost(0);
 4700   format %{ %}
 4701   interface(CONST_INTER);
 4702 %}
 4703 
 4704 // 32 bit integer valid for vector add sub immediate
 4705 operand immIAddSubV()
 4706 %{
 4707   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4708   match(ConI);
 4709 
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(CONST_INTER);
 4713 %}
 4714 
 4715 // 32 bit unsigned integer valid for logical immediate
 4716 
 4717 operand immBLog()
 4718 %{
 4719   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4720   match(ConI);
 4721 
 4722   op_cost(0);
 4723   format %{ %}
 4724   interface(CONST_INTER);
 4725 %}
 4726 
 4727 operand immSLog()
 4728 %{
 4729   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4730   match(ConI);
 4731 
 4732   op_cost(0);
 4733   format %{ %}
 4734   interface(CONST_INTER);
 4735 %}
 4736 
 4737 operand immILog()
 4738 %{
 4739   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4740   match(ConI);
 4741 
 4742   op_cost(0);
 4743   format %{ %}
 4744   interface(CONST_INTER);
 4745 %}
 4746 
 4747 // Integer operands 64 bit
 4748 // 64 bit immediate
 4749 operand immL()
 4750 %{
 4751   match(ConL);
 4752 
 4753   op_cost(0);
 4754   format %{ %}
 4755   interface(CONST_INTER);
 4756 %}
 4757 
 4758 // 64 bit zero
 4759 operand immL0()
 4760 %{
 4761   predicate(n->get_long() == 0);
 4762   match(ConL);
 4763 
 4764   op_cost(0);
 4765   format %{ %}
 4766   interface(CONST_INTER);
 4767 %}
 4768 
 4769 // 64 bit unit increment
 4770 operand immL_1()
 4771 %{
 4772   predicate(n->get_long() == 1);
 4773   match(ConL);
 4774 
 4775   op_cost(0);
 4776   format %{ %}
 4777   interface(CONST_INTER);
 4778 %}
 4779 
 4780 // 64 bit unit decrement
 4781 operand immL_M1()
 4782 %{
 4783   predicate(n->get_long() == -1);
 4784   match(ConL);
 4785 
 4786   op_cost(0);
 4787   format %{ %}
 4788   interface(CONST_INTER);
 4789 %}
 4790 
 4791 // 32 bit offset of pc in thread anchor
 4792 
 4793 operand immL_pc_off()
 4794 %{
 4795   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4796                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4797   match(ConL);
 4798 
 4799   op_cost(0);
 4800   format %{ %}
 4801   interface(CONST_INTER);
 4802 %}
 4803 
 4804 // 64 bit integer valid for add sub immediate
 4805 operand immLAddSub()
 4806 %{
 4807   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4808   match(ConL);
 4809   op_cost(0);
 4810   format %{ %}
 4811   interface(CONST_INTER);
 4812 %}
 4813 
 4814 // 64 bit integer valid for addv subv immediate
 4815 operand immLAddSubV()
 4816 %{
 4817   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4818   match(ConL);
 4819 
 4820   op_cost(0);
 4821   format %{ %}
 4822   interface(CONST_INTER);
 4823 %}
 4824 
 4825 // 64 bit integer valid for logical immediate
 4826 operand immLLog()
 4827 %{
 4828   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4829   match(ConL);
 4830   op_cost(0);
 4831   format %{ %}
 4832   interface(CONST_INTER);
 4833 %}
 4834 
 4835 // Long Immediate: low 32-bit mask
 4836 operand immL_32bits()
 4837 %{
 4838   predicate(n->get_long() == 0xFFFFFFFFL);
 4839   match(ConL);
 4840   op_cost(0);
 4841   format %{ %}
 4842   interface(CONST_INTER);
 4843 %}
 4844 
 4845 // Pointer operands
 4846 // Pointer Immediate
 4847 operand immP()
 4848 %{
 4849   match(ConP);
 4850 
 4851   op_cost(0);
 4852   format %{ %}
 4853   interface(CONST_INTER);
 4854 %}
 4855 
 4856 // NULL Pointer Immediate
 4857 operand immP0()
 4858 %{
 4859   predicate(n->get_ptr() == 0);
 4860   match(ConP);
 4861 
 4862   op_cost(0);
 4863   format %{ %}
 4864   interface(CONST_INTER);
 4865 %}
 4866 
 4867 // Pointer Immediate One
 4868 // this is used in object initialization (initial object header)
 4869 operand immP_1()
 4870 %{
 4871   predicate(n->get_ptr() == 1);
 4872   match(ConP);
 4873 
 4874   op_cost(0);
 4875   format %{ %}
 4876   interface(CONST_INTER);
 4877 %}
 4878 
 4879 // Card Table Byte Map Base
 4880 operand immByteMapBase()
 4881 %{
 4882   // Get base of card map
 4883   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4884             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4885   match(ConP);
 4886 
 4887   op_cost(0);
 4888   format %{ %}
 4889   interface(CONST_INTER);
 4890 %}
 4891 
 4892 // Pointer Immediate Minus One
 4893 // this is used when we want to write the current PC to the thread anchor
 4894 operand immP_M1()
 4895 %{
 4896   predicate(n->get_ptr() == -1);
 4897   match(ConP);
 4898 
 4899   op_cost(0);
 4900   format %{ %}
 4901   interface(CONST_INTER);
 4902 %}
 4903 
 4904 // Pointer Immediate Minus Two
 4905 // this is used when we want to write the current PC to the thread anchor
 4906 operand immP_M2()
 4907 %{
 4908   predicate(n->get_ptr() == -2);
 4909   match(ConP);
 4910 
 4911   op_cost(0);
 4912   format %{ %}
 4913   interface(CONST_INTER);
 4914 %}
 4915 
 4916 // Float and Double operands
 4917 // Double Immediate
 4918 operand immD()
 4919 %{
 4920   match(ConD);
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(CONST_INTER);
 4924 %}
 4925 
 4926 // Double Immediate: +0.0d
 4927 operand immD0()
 4928 %{
 4929   predicate(jlong_cast(n->getd()) == 0);
 4930   match(ConD);
 4931 
 4932   op_cost(0);
 4933   format %{ %}
 4934   interface(CONST_INTER);
 4935 %}
 4936 
 4937 // constant 'double +0.0'.
 4938 operand immDPacked()
 4939 %{
 4940   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4941   match(ConD);
 4942   op_cost(0);
 4943   format %{ %}
 4944   interface(CONST_INTER);
 4945 %}
 4946 
 4947 // Float Immediate
 4948 operand immF()
 4949 %{
 4950   match(ConF);
 4951   op_cost(0);
 4952   format %{ %}
 4953   interface(CONST_INTER);
 4954 %}
 4955 
 4956 // Float Immediate: +0.0f.
 4957 operand immF0()
 4958 %{
 4959   predicate(jint_cast(n->getf()) == 0);
 4960   match(ConF);
 4961 
 4962   op_cost(0);
 4963   format %{ %}
 4964   interface(CONST_INTER);
 4965 %}
 4966 
 4967 //
 4968 operand immFPacked()
 4969 %{
 4970   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4971   match(ConF);
 4972   op_cost(0);
 4973   format %{ %}
 4974   interface(CONST_INTER);
 4975 %}
 4976 
 4977 // Narrow pointer operands
 4978 // Narrow Pointer Immediate
 4979 operand immN()
 4980 %{
 4981   match(ConN);
 4982 
 4983   op_cost(0);
 4984   format %{ %}
 4985   interface(CONST_INTER);
 4986 %}
 4987 
 4988 // Narrow NULL Pointer Immediate
 4989 operand immN0()
 4990 %{
 4991   predicate(n->get_narrowcon() == 0);
 4992   match(ConN);
 4993 
 4994   op_cost(0);
 4995   format %{ %}
 4996   interface(CONST_INTER);
 4997 %}
 4998 
 4999 operand immNKlass()
 5000 %{
 5001   match(ConNKlass);
 5002 
 5003   op_cost(0);
 5004   format %{ %}
 5005   interface(CONST_INTER);
 5006 %}
 5007 
 5008 // Integer 32 bit Register Operands
 5009 // Integer 32 bitRegister (excludes SP)
 5010 operand iRegI()
 5011 %{
 5012   constraint(ALLOC_IN_RC(any_reg32));
 5013   match(RegI);
 5014   match(iRegINoSp);
 5015   op_cost(0);
 5016   format %{ %}
 5017   interface(REG_INTER);
 5018 %}
 5019 
 5020 // Integer 32 bit Register not Special
 5021 operand iRegINoSp()
 5022 %{
 5023   constraint(ALLOC_IN_RC(no_special_reg32));
 5024   match(RegI);
 5025   op_cost(0);
 5026   format %{ %}
 5027   interface(REG_INTER);
 5028 %}
 5029 
 5030 // Integer 64 bit Register Operands
 5031 // Integer 64 bit Register (includes SP)
 5032 operand iRegL()
 5033 %{
 5034   constraint(ALLOC_IN_RC(any_reg));
 5035   match(RegL);
 5036   match(iRegLNoSp);
 5037   op_cost(0);
 5038   format %{ %}
 5039   interface(REG_INTER);
 5040 %}
 5041 
 5042 // Integer 64 bit Register not Special
 5043 operand iRegLNoSp()
 5044 %{
 5045   constraint(ALLOC_IN_RC(no_special_reg));
 5046   match(RegL);
 5047   match(iRegL_R0);
 5048   format %{ %}
 5049   interface(REG_INTER);
 5050 %}
 5051 
 5052 // Pointer Register Operands
 5053 // Pointer Register
 5054 operand iRegP()
 5055 %{
 5056   constraint(ALLOC_IN_RC(ptr_reg));
 5057   match(RegP);
 5058   match(iRegPNoSp);
 5059   match(iRegP_R0);
 5060   //match(iRegP_R2);
 5061   //match(iRegP_R4);
 5062   //match(iRegP_R5);
 5063   match(thread_RegP);
 5064   op_cost(0);
 5065   format %{ %}
 5066   interface(REG_INTER);
 5067 %}
 5068 
 5069 // Pointer 64 bit Register not Special
 5070 operand iRegPNoSp()
 5071 %{
 5072   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5073   match(RegP);
 5074   // match(iRegP);
 5075   // match(iRegP_R0);
 5076   // match(iRegP_R2);
 5077   // match(iRegP_R4);
 5078   // match(iRegP_R5);
 5079   // match(thread_RegP);
 5080   op_cost(0);
 5081   format %{ %}
 5082   interface(REG_INTER);
 5083 %}
 5084 
 5085 // Pointer 64 bit Register R0 only
 5086 operand iRegP_R0()
 5087 %{
 5088   constraint(ALLOC_IN_RC(r0_reg));
 5089   match(RegP);
 5090   // match(iRegP);
 5091   match(iRegPNoSp);
 5092   op_cost(0);
 5093   format %{ %}
 5094   interface(REG_INTER);
 5095 %}
 5096 
 5097 // Pointer 64 bit Register R1 only
 5098 operand iRegP_R1()
 5099 %{
 5100   constraint(ALLOC_IN_RC(r1_reg));
 5101   match(RegP);
 5102   // match(iRegP);
 5103   match(iRegPNoSp);
 5104   op_cost(0);
 5105   format %{ %}
 5106   interface(REG_INTER);
 5107 %}
 5108 
 5109 // Pointer 64 bit Register R2 only
 5110 operand iRegP_R2()
 5111 %{
 5112   constraint(ALLOC_IN_RC(r2_reg));
 5113   match(RegP);
 5114   // match(iRegP);
 5115   match(iRegPNoSp);
 5116   op_cost(0);
 5117   format %{ %}
 5118   interface(REG_INTER);
 5119 %}
 5120 
 5121 // Pointer 64 bit Register R3 only
 5122 operand iRegP_R3()
 5123 %{
 5124   constraint(ALLOC_IN_RC(r3_reg));
 5125   match(RegP);
 5126   // match(iRegP);
 5127   match(iRegPNoSp);
 5128   op_cost(0);
 5129   format %{ %}
 5130   interface(REG_INTER);
 5131 %}
 5132 
 5133 // Pointer 64 bit Register R4 only
 5134 operand iRegP_R4()
 5135 %{
 5136   constraint(ALLOC_IN_RC(r4_reg));
 5137   match(RegP);
 5138   // match(iRegP);
 5139   match(iRegPNoSp);
 5140   op_cost(0);
 5141   format %{ %}
 5142   interface(REG_INTER);
 5143 %}
 5144 
 5145 // Pointer 64 bit Register R5 only
 5146 operand iRegP_R5()
 5147 %{
 5148   constraint(ALLOC_IN_RC(r5_reg));
 5149   match(RegP);
 5150   // match(iRegP);
 5151   match(iRegPNoSp);
 5152   op_cost(0);
 5153   format %{ %}
 5154   interface(REG_INTER);
 5155 %}
 5156 
 5157 // Pointer 64 bit Register R10 only
 5158 operand iRegP_R10()
 5159 %{
 5160   constraint(ALLOC_IN_RC(r10_reg));
 5161   match(RegP);
 5162   // match(iRegP);
 5163   match(iRegPNoSp);
 5164   op_cost(0);
 5165   format %{ %}
 5166   interface(REG_INTER);
 5167 %}
 5168 
 5169 // Long 64 bit Register R0 only
 5170 operand iRegL_R0()
 5171 %{
 5172   constraint(ALLOC_IN_RC(r0_reg));
 5173   match(RegL);
 5174   match(iRegLNoSp);
 5175   op_cost(0);
 5176   format %{ %}
 5177   interface(REG_INTER);
 5178 %}
 5179 
 5180 // Long 64 bit Register R2 only
 5181 operand iRegL_R2()
 5182 %{
 5183   constraint(ALLOC_IN_RC(r2_reg));
 5184   match(RegL);
 5185   match(iRegLNoSp);
 5186   op_cost(0);
 5187   format %{ %}
 5188   interface(REG_INTER);
 5189 %}
 5190 
 5191 // Long 64 bit Register R3 only
 5192 operand iRegL_R3()
 5193 %{
 5194   constraint(ALLOC_IN_RC(r3_reg));
 5195   match(RegL);
 5196   match(iRegLNoSp);
 5197   op_cost(0);
 5198   format %{ %}
 5199   interface(REG_INTER);
 5200 %}
 5201 
 5202 // Long 64 bit Register R11 only
 5203 operand iRegL_R11()
 5204 %{
 5205   constraint(ALLOC_IN_RC(r11_reg));
 5206   match(RegL);
 5207   match(iRegLNoSp);
 5208   op_cost(0);
 5209   format %{ %}
 5210   interface(REG_INTER);
 5211 %}
 5212 
 5213 // Pointer 64 bit Register FP only
 5214 operand iRegP_FP()
 5215 %{
 5216   constraint(ALLOC_IN_RC(fp_reg));
 5217   match(RegP);
 5218   // match(iRegP);
 5219   op_cost(0);
 5220   format %{ %}
 5221   interface(REG_INTER);
 5222 %}
 5223 
 5224 // Register R0 only
 5225 operand iRegI_R0()
 5226 %{
 5227   constraint(ALLOC_IN_RC(int_r0_reg));
 5228   match(RegI);
 5229   match(iRegINoSp);
 5230   op_cost(0);
 5231   format %{ %}
 5232   interface(REG_INTER);
 5233 %}
 5234 
 5235 // Register R2 only
 5236 operand iRegI_R2()
 5237 %{
 5238   constraint(ALLOC_IN_RC(int_r2_reg));
 5239   match(RegI);
 5240   match(iRegINoSp);
 5241   op_cost(0);
 5242   format %{ %}
 5243   interface(REG_INTER);
 5244 %}
 5245 
 5246 // Register R3 only
 5247 operand iRegI_R3()
 5248 %{
 5249   constraint(ALLOC_IN_RC(int_r3_reg));
 5250   match(RegI);
 5251   match(iRegINoSp);
 5252   op_cost(0);
 5253   format %{ %}
 5254   interface(REG_INTER);
 5255 %}
 5256 
 5257 
 5258 // Register R4 only
 5259 operand iRegI_R4()
 5260 %{
 5261   constraint(ALLOC_IN_RC(int_r4_reg));
 5262   match(RegI);
 5263   match(iRegINoSp);
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 
 5270 // Pointer Register Operands
 5271 // Narrow Pointer Register
 5272 operand iRegN()
 5273 %{
 5274   constraint(ALLOC_IN_RC(any_reg32));
 5275   match(RegN);
 5276   match(iRegNNoSp);
 5277   op_cost(0);
 5278   format %{ %}
 5279   interface(REG_INTER);
 5280 %}
 5281 
 5282 operand iRegN_R0()
 5283 %{
 5284   constraint(ALLOC_IN_RC(r0_reg));
 5285   match(iRegN);
 5286   op_cost(0);
 5287   format %{ %}
 5288   interface(REG_INTER);
 5289 %}
 5290 
 5291 operand iRegN_R2()
 5292 %{
 5293   constraint(ALLOC_IN_RC(r2_reg));
 5294   match(iRegN);
 5295   op_cost(0);
 5296   format %{ %}
 5297   interface(REG_INTER);
 5298 %}
 5299 
 5300 operand iRegN_R3()
 5301 %{
 5302   constraint(ALLOC_IN_RC(r3_reg));
 5303   match(iRegN);
 5304   op_cost(0);
 5305   format %{ %}
 5306   interface(REG_INTER);
 5307 %}
 5308 
 5309 // Integer 64 bit Register not Special
 5310 operand iRegNNoSp()
 5311 %{
 5312   constraint(ALLOC_IN_RC(no_special_reg32));
 5313   match(RegN);
 5314   op_cost(0);
 5315   format %{ %}
 5316   interface(REG_INTER);
 5317 %}
 5318 
 5319 // heap base register -- used for encoding immN0
 5320 
 5321 operand iRegIHeapbase()
 5322 %{
 5323   constraint(ALLOC_IN_RC(heapbase_reg));
 5324   match(RegI);
 5325   op_cost(0);
 5326   format %{ %}
 5327   interface(REG_INTER);
 5328 %}
 5329 
 5330 // Float Register
 5331 // Float register operands
 5332 operand vRegF()
 5333 %{
 5334   constraint(ALLOC_IN_RC(float_reg));
 5335   match(RegF);
 5336 
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 // Double Register
 5343 // Double register operands
 5344 operand vRegD()
 5345 %{
 5346   constraint(ALLOC_IN_RC(double_reg));
 5347   match(RegD);
 5348 
 5349   op_cost(0);
 5350   format %{ %}
 5351   interface(REG_INTER);
 5352 %}
 5353 
 5354 // Generic vector class. This will be used for
 5355 // all vector operands, including NEON and SVE,
 5356 // but currently only used for SVE VecA.
 5357 operand vReg()
 5358 %{
 5359   constraint(ALLOC_IN_RC(vectora_reg));
 5360   match(VecA);
 5361   op_cost(0);
 5362   format %{ %}
 5363   interface(REG_INTER);
 5364 %}
 5365 
 5366 operand vecD()
 5367 %{
 5368   constraint(ALLOC_IN_RC(vectord_reg));
 5369   match(VecD);
 5370 
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vecX()
 5377 %{
 5378   constraint(ALLOC_IN_RC(vectorx_reg));
 5379   match(VecX);
 5380 
 5381   op_cost(0);
 5382   format %{ %}
 5383   interface(REG_INTER);
 5384 %}
 5385 
 5386 operand vRegD_V0()
 5387 %{
 5388   constraint(ALLOC_IN_RC(v0_reg));
 5389   match(RegD);
 5390   op_cost(0);
 5391   format %{ %}
 5392   interface(REG_INTER);
 5393 %}
 5394 
 5395 operand vRegD_V1()
 5396 %{
 5397   constraint(ALLOC_IN_RC(v1_reg));
 5398   match(RegD);
 5399   op_cost(0);
 5400   format %{ %}
 5401   interface(REG_INTER);
 5402 %}
 5403 
 5404 operand vRegD_V2()
 5405 %{
 5406   constraint(ALLOC_IN_RC(v2_reg));
 5407   match(RegD);
 5408   op_cost(0);
 5409   format %{ %}
 5410   interface(REG_INTER);
 5411 %}
 5412 
 5413 operand vRegD_V3()
 5414 %{
 5415   constraint(ALLOC_IN_RC(v3_reg));
 5416   match(RegD);
 5417   op_cost(0);
 5418   format %{ %}
 5419   interface(REG_INTER);
 5420 %}
 5421 
 5422 operand vRegD_V4()
 5423 %{
 5424   constraint(ALLOC_IN_RC(v4_reg));
 5425   match(RegD);
 5426   op_cost(0);
 5427   format %{ %}
 5428   interface(REG_INTER);
 5429 %}
 5430 
 5431 operand vRegD_V5()
 5432 %{
 5433   constraint(ALLOC_IN_RC(v5_reg));
 5434   match(RegD);
 5435   op_cost(0);
 5436   format %{ %}
 5437   interface(REG_INTER);
 5438 %}
 5439 
 5440 operand vRegD_V6()
 5441 %{
 5442   constraint(ALLOC_IN_RC(v6_reg));
 5443   match(RegD);
 5444   op_cost(0);
 5445   format %{ %}
 5446   interface(REG_INTER);
 5447 %}
 5448 
 5449 operand vRegD_V7()
 5450 %{
 5451   constraint(ALLOC_IN_RC(v7_reg));
 5452   match(RegD);
 5453   op_cost(0);
 5454   format %{ %}
 5455   interface(REG_INTER);
 5456 %}
 5457 
 5458 operand vRegD_V8()
 5459 %{
 5460   constraint(ALLOC_IN_RC(v8_reg));
 5461   match(RegD);
 5462   op_cost(0);
 5463   format %{ %}
 5464   interface(REG_INTER);
 5465 %}
 5466 
 5467 operand vRegD_V9()
 5468 %{
 5469   constraint(ALLOC_IN_RC(v9_reg));
 5470   match(RegD);
 5471   op_cost(0);
 5472   format %{ %}
 5473   interface(REG_INTER);
 5474 %}
 5475 
 5476 operand vRegD_V10()
 5477 %{
 5478   constraint(ALLOC_IN_RC(v10_reg));
 5479   match(RegD);
 5480   op_cost(0);
 5481   format %{ %}
 5482   interface(REG_INTER);
 5483 %}
 5484 
 5485 operand vRegD_V11()
 5486 %{
 5487   constraint(ALLOC_IN_RC(v11_reg));
 5488   match(RegD);
 5489   op_cost(0);
 5490   format %{ %}
 5491   interface(REG_INTER);
 5492 %}
 5493 
 5494 operand vRegD_V12()
 5495 %{
 5496   constraint(ALLOC_IN_RC(v12_reg));
 5497   match(RegD);
 5498   op_cost(0);
 5499   format %{ %}
 5500   interface(REG_INTER);
 5501 %}
 5502 
 5503 operand vRegD_V13()
 5504 %{
 5505   constraint(ALLOC_IN_RC(v13_reg));
 5506   match(RegD);
 5507   op_cost(0);
 5508   format %{ %}
 5509   interface(REG_INTER);
 5510 %}
 5511 
 5512 operand vRegD_V14()
 5513 %{
 5514   constraint(ALLOC_IN_RC(v14_reg));
 5515   match(RegD);
 5516   op_cost(0);
 5517   format %{ %}
 5518   interface(REG_INTER);
 5519 %}
 5520 
 5521 operand vRegD_V15()
 5522 %{
 5523   constraint(ALLOC_IN_RC(v15_reg));
 5524   match(RegD);
 5525   op_cost(0);
 5526   format %{ %}
 5527   interface(REG_INTER);
 5528 %}
 5529 
 5530 operand vRegD_V16()
 5531 %{
 5532   constraint(ALLOC_IN_RC(v16_reg));
 5533   match(RegD);
 5534   op_cost(0);
 5535   format %{ %}
 5536   interface(REG_INTER);
 5537 %}
 5538 
 5539 operand vRegD_V17()
 5540 %{
 5541   constraint(ALLOC_IN_RC(v17_reg));
 5542   match(RegD);
 5543   op_cost(0);
 5544   format %{ %}
 5545   interface(REG_INTER);
 5546 %}
 5547 
 5548 operand vRegD_V18()
 5549 %{
 5550   constraint(ALLOC_IN_RC(v18_reg));
 5551   match(RegD);
 5552   op_cost(0);
 5553   format %{ %}
 5554   interface(REG_INTER);
 5555 %}
 5556 
 5557 operand vRegD_V19()
 5558 %{
 5559   constraint(ALLOC_IN_RC(v19_reg));
 5560   match(RegD);
 5561   op_cost(0);
 5562   format %{ %}
 5563   interface(REG_INTER);
 5564 %}
 5565 
 5566 operand vRegD_V20()
 5567 %{
 5568   constraint(ALLOC_IN_RC(v20_reg));
 5569   match(RegD);
 5570   op_cost(0);
 5571   format %{ %}
 5572   interface(REG_INTER);
 5573 %}
 5574 
 5575 operand vRegD_V21()
 5576 %{
 5577   constraint(ALLOC_IN_RC(v21_reg));
 5578   match(RegD);
 5579   op_cost(0);
 5580   format %{ %}
 5581   interface(REG_INTER);
 5582 %}
 5583 
 5584 operand vRegD_V22()
 5585 %{
 5586   constraint(ALLOC_IN_RC(v22_reg));
 5587   match(RegD);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 operand vRegD_V23()
 5594 %{
 5595   constraint(ALLOC_IN_RC(v23_reg));
 5596   match(RegD);
 5597   op_cost(0);
 5598   format %{ %}
 5599   interface(REG_INTER);
 5600 %}
 5601 
 5602 operand vRegD_V24()
 5603 %{
 5604   constraint(ALLOC_IN_RC(v24_reg));
 5605   match(RegD);
 5606   op_cost(0);
 5607   format %{ %}
 5608   interface(REG_INTER);
 5609 %}
 5610 
 5611 operand vRegD_V25()
 5612 %{
 5613   constraint(ALLOC_IN_RC(v25_reg));
 5614   match(RegD);
 5615   op_cost(0);
 5616   format %{ %}
 5617   interface(REG_INTER);
 5618 %}
 5619 
 5620 operand vRegD_V26()
 5621 %{
 5622   constraint(ALLOC_IN_RC(v26_reg));
 5623   match(RegD);
 5624   op_cost(0);
 5625   format %{ %}
 5626   interface(REG_INTER);
 5627 %}
 5628 
 5629 operand vRegD_V27()
 5630 %{
 5631   constraint(ALLOC_IN_RC(v27_reg));
 5632   match(RegD);
 5633   op_cost(0);
 5634   format %{ %}
 5635   interface(REG_INTER);
 5636 %}
 5637 
 5638 operand vRegD_V28()
 5639 %{
 5640   constraint(ALLOC_IN_RC(v28_reg));
 5641   match(RegD);
 5642   op_cost(0);
 5643   format %{ %}
 5644   interface(REG_INTER);
 5645 %}
 5646 
 5647 operand vRegD_V29()
 5648 %{
 5649   constraint(ALLOC_IN_RC(v29_reg));
 5650   match(RegD);
 5651   op_cost(0);
 5652   format %{ %}
 5653   interface(REG_INTER);
 5654 %}
 5655 
 5656 operand vRegD_V30()
 5657 %{
 5658   constraint(ALLOC_IN_RC(v30_reg));
 5659   match(RegD);
 5660   op_cost(0);
 5661   format %{ %}
 5662   interface(REG_INTER);
 5663 %}
 5664 
 5665 operand vRegD_V31()
 5666 %{
 5667   constraint(ALLOC_IN_RC(v31_reg));
 5668   match(RegD);
 5669   op_cost(0);
 5670   format %{ %}
 5671   interface(REG_INTER);
 5672 %}
 5673 
 5674 operand pReg()
 5675 %{
 5676   constraint(ALLOC_IN_RC(pr_reg));
 5677   match(RegVectMask);
 5678   match(pRegGov);
 5679   op_cost(0);
 5680   format %{ %}
 5681   interface(REG_INTER);
 5682 %}
 5683 
 5684 operand pRegGov()
 5685 %{
 5686   constraint(ALLOC_IN_RC(gov_pr));
 5687   match(RegVectMask);
 5688   op_cost(0);
 5689   format %{ %}
 5690   interface(REG_INTER);
 5691 %}
 5692 
 5693 operand pRegGov_P0()
 5694 %{
 5695   constraint(ALLOC_IN_RC(p0_reg));
 5696   match(RegVectMask);
 5697   op_cost(0);
 5698   format %{ %}
 5699   interface(REG_INTER);
 5700 %}
 5701 
 5702 operand pRegGov_P1()
 5703 %{
 5704   constraint(ALLOC_IN_RC(p1_reg));
 5705   match(RegVectMask);
 5706   op_cost(0);
 5707   format %{ %}
 5708   interface(REG_INTER);
 5709 %}
 5710 
 5711 // Flags register, used as output of signed compare instructions
 5712 
 5713 // note that on AArch64 we also use this register as the output for
 5714 // for floating point compare instructions (CmpF CmpD). this ensures
 5715 // that ordered inequality tests use GT, GE, LT or LE none of which
 5716 // pass through cases where the result is unordered i.e. one or both
 5717 // inputs to the compare is a NaN. this means that the ideal code can
 5718 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5719 // (where the comparison should always fail). EQ and NE tests are
 5720 // always generated in ideal code so that unordered folds into the NE
 5721 // case, matching the behaviour of AArch64 NE.
 5722 //
 5723 // This differs from x86 where the outputs of FP compares use a
 5724 // special FP flags registers and where compares based on this
 5725 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5726 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5727 // to explicitly handle the unordered case in branches. x86 also has
 5728 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5729 
 5730 operand rFlagsReg()
 5731 %{
 5732   constraint(ALLOC_IN_RC(int_flags));
 5733   match(RegFlags);
 5734 
 5735   op_cost(0);
 5736   format %{ "RFLAGS" %}
 5737   interface(REG_INTER);
 5738 %}
 5739 
 5740 // Flags register, used as output of unsigned compare instructions
 5741 operand rFlagsRegU()
 5742 %{
 5743   constraint(ALLOC_IN_RC(int_flags));
 5744   match(RegFlags);
 5745 
 5746   op_cost(0);
 5747   format %{ "RFLAGSU" %}
 5748   interface(REG_INTER);
 5749 %}
 5750 
 5751 // Special Registers
 5752 
 5753 // Method Register
 5754 operand inline_cache_RegP(iRegP reg)
 5755 %{
 5756   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5757   match(reg);
 5758   match(iRegPNoSp);
 5759   op_cost(0);
 5760   format %{ %}
 5761   interface(REG_INTER);
 5762 %}
 5763 
 5764 // Thread Register
 5765 operand thread_RegP(iRegP reg)
 5766 %{
 5767   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5768   match(reg);
 5769   op_cost(0);
 5770   format %{ %}
 5771   interface(REG_INTER);
 5772 %}
 5773 
 5774 operand lr_RegP(iRegP reg)
 5775 %{
 5776   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5777   match(reg);
 5778   op_cost(0);
 5779   format %{ %}
 5780   interface(REG_INTER);
 5781 %}
 5782 
 5783 //----------Memory Operands----------------------------------------------------
 5784 
 5785 operand indirect(iRegP reg)
 5786 %{
 5787   constraint(ALLOC_IN_RC(ptr_reg));
 5788   match(reg);
 5789   op_cost(0);
 5790   format %{ "[$reg]" %}
 5791   interface(MEMORY_INTER) %{
 5792     base($reg);
 5793     index(0xffffffff);
 5794     scale(0x0);
 5795     disp(0x0);
 5796   %}
 5797 %}
 5798 
 5799 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5800 %{
 5801   constraint(ALLOC_IN_RC(ptr_reg));
 5802   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5803   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5804   op_cost(0);
 5805   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5806   interface(MEMORY_INTER) %{
 5807     base($reg);
 5808     index($ireg);
 5809     scale($scale);
 5810     disp(0x0);
 5811   %}
 5812 %}
 5813 
 5814 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5815 %{
 5816   constraint(ALLOC_IN_RC(ptr_reg));
 5817   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5818   match(AddP reg (LShiftL lreg scale));
 5819   op_cost(0);
 5820   format %{ "$reg, $lreg lsl($scale)" %}
 5821   interface(MEMORY_INTER) %{
 5822     base($reg);
 5823     index($lreg);
 5824     scale($scale);
 5825     disp(0x0);
 5826   %}
 5827 %}
 5828 
 5829 operand indIndexI2L(iRegP reg, iRegI ireg)
 5830 %{
 5831   constraint(ALLOC_IN_RC(ptr_reg));
 5832   match(AddP reg (ConvI2L ireg));
 5833   op_cost(0);
 5834   format %{ "$reg, $ireg, 0, I2L" %}
 5835   interface(MEMORY_INTER) %{
 5836     base($reg);
 5837     index($ireg);
 5838     scale(0x0);
 5839     disp(0x0);
 5840   %}
 5841 %}
 5842 
 5843 operand indIndex(iRegP reg, iRegL lreg)
 5844 %{
 5845   constraint(ALLOC_IN_RC(ptr_reg));
 5846   match(AddP reg lreg);
 5847   op_cost(0);
 5848   format %{ "$reg, $lreg" %}
 5849   interface(MEMORY_INTER) %{
 5850     base($reg);
 5851     index($lreg);
 5852     scale(0x0);
 5853     disp(0x0);
 5854   %}
 5855 %}
 5856 
 5857 operand indOffI(iRegP reg, immIOffset off)
 5858 %{
 5859   constraint(ALLOC_IN_RC(ptr_reg));
 5860   match(AddP reg off);
 5861   op_cost(0);
 5862   format %{ "[$reg, $off]" %}
 5863   interface(MEMORY_INTER) %{
 5864     base($reg);
 5865     index(0xffffffff);
 5866     scale(0x0);
 5867     disp($off);
 5868   %}
 5869 %}
 5870 
 5871 operand indOffI1(iRegP reg, immIOffset1 off)
 5872 %{
 5873   constraint(ALLOC_IN_RC(ptr_reg));
 5874   match(AddP reg off);
 5875   op_cost(0);
 5876   format %{ "[$reg, $off]" %}
 5877   interface(MEMORY_INTER) %{
 5878     base($reg);
 5879     index(0xffffffff);
 5880     scale(0x0);
 5881     disp($off);
 5882   %}
 5883 %}
 5884 
 5885 operand indOffI2(iRegP reg, immIOffset2 off)
 5886 %{
 5887   constraint(ALLOC_IN_RC(ptr_reg));
 5888   match(AddP reg off);
 5889   op_cost(0);
 5890   format %{ "[$reg, $off]" %}
 5891   interface(MEMORY_INTER) %{
 5892     base($reg);
 5893     index(0xffffffff);
 5894     scale(0x0);
 5895     disp($off);
 5896   %}
 5897 %}
 5898 
 5899 operand indOffI4(iRegP reg, immIOffset4 off)
 5900 %{
 5901   constraint(ALLOC_IN_RC(ptr_reg));
 5902   match(AddP reg off);
 5903   op_cost(0);
 5904   format %{ "[$reg, $off]" %}
 5905   interface(MEMORY_INTER) %{
 5906     base($reg);
 5907     index(0xffffffff);
 5908     scale(0x0);
 5909     disp($off);
 5910   %}
 5911 %}
 5912 
 5913 operand indOffI8(iRegP reg, immIOffset8 off)
 5914 %{
 5915   constraint(ALLOC_IN_RC(ptr_reg));
 5916   match(AddP reg off);
 5917   op_cost(0);
 5918   format %{ "[$reg, $off]" %}
 5919   interface(MEMORY_INTER) %{
 5920     base($reg);
 5921     index(0xffffffff);
 5922     scale(0x0);
 5923     disp($off);
 5924   %}
 5925 %}
 5926 
 5927 operand indOffI16(iRegP reg, immIOffset16 off)
 5928 %{
 5929   constraint(ALLOC_IN_RC(ptr_reg));
 5930   match(AddP reg off);
 5931   op_cost(0);
 5932   format %{ "[$reg, $off]" %}
 5933   interface(MEMORY_INTER) %{
 5934     base($reg);
 5935     index(0xffffffff);
 5936     scale(0x0);
 5937     disp($off);
 5938   %}
 5939 %}
 5940 
 5941 operand indOffL(iRegP reg, immLoffset off)
 5942 %{
 5943   constraint(ALLOC_IN_RC(ptr_reg));
 5944   match(AddP reg off);
 5945   op_cost(0);
 5946   format %{ "[$reg, $off]" %}
 5947   interface(MEMORY_INTER) %{
 5948     base($reg);
 5949     index(0xffffffff);
 5950     scale(0x0);
 5951     disp($off);
 5952   %}
 5953 %}
 5954 
 5955 operand indOffL1(iRegP reg, immLoffset1 off)
 5956 %{
 5957   constraint(ALLOC_IN_RC(ptr_reg));
 5958   match(AddP reg off);
 5959   op_cost(0);
 5960   format %{ "[$reg, $off]" %}
 5961   interface(MEMORY_INTER) %{
 5962     base($reg);
 5963     index(0xffffffff);
 5964     scale(0x0);
 5965     disp($off);
 5966   %}
 5967 %}
 5968 
 5969 operand indOffL2(iRegP reg, immLoffset2 off)
 5970 %{
 5971   constraint(ALLOC_IN_RC(ptr_reg));
 5972   match(AddP reg off);
 5973   op_cost(0);
 5974   format %{ "[$reg, $off]" %}
 5975   interface(MEMORY_INTER) %{
 5976     base($reg);
 5977     index(0xffffffff);
 5978     scale(0x0);
 5979     disp($off);
 5980   %}
 5981 %}
 5982 
 5983 operand indOffL4(iRegP reg, immLoffset4 off)
 5984 %{
 5985   constraint(ALLOC_IN_RC(ptr_reg));
 5986   match(AddP reg off);
 5987   op_cost(0);
 5988   format %{ "[$reg, $off]" %}
 5989   interface(MEMORY_INTER) %{
 5990     base($reg);
 5991     index(0xffffffff);
 5992     scale(0x0);
 5993     disp($off);
 5994   %}
 5995 %}
 5996 
 5997 operand indOffL8(iRegP reg, immLoffset8 off)
 5998 %{
 5999   constraint(ALLOC_IN_RC(ptr_reg));
 6000   match(AddP reg off);
 6001   op_cost(0);
 6002   format %{ "[$reg, $off]" %}
 6003   interface(MEMORY_INTER) %{
 6004     base($reg);
 6005     index(0xffffffff);
 6006     scale(0x0);
 6007     disp($off);
 6008   %}
 6009 %}
 6010 
 6011 operand indOffL16(iRegP reg, immLoffset16 off)
 6012 %{
 6013   constraint(ALLOC_IN_RC(ptr_reg));
 6014   match(AddP reg off);
 6015   op_cost(0);
 6016   format %{ "[$reg, $off]" %}
 6017   interface(MEMORY_INTER) %{
 6018     base($reg);
 6019     index(0xffffffff);
 6020     scale(0x0);
 6021     disp($off);
 6022   %}
 6023 %}
 6024 
 6025 operand indirectN(iRegN reg)
 6026 %{
 6027   predicate(CompressedOops::shift() == 0);
 6028   constraint(ALLOC_IN_RC(ptr_reg));
 6029   match(DecodeN reg);
 6030   op_cost(0);
 6031   format %{ "[$reg]\t# narrow" %}
 6032   interface(MEMORY_INTER) %{
 6033     base($reg);
 6034     index(0xffffffff);
 6035     scale(0x0);
 6036     disp(0x0);
 6037   %}
 6038 %}
 6039 
 6040 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6041 %{
 6042   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6043   constraint(ALLOC_IN_RC(ptr_reg));
 6044   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6045   op_cost(0);
 6046   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6047   interface(MEMORY_INTER) %{
 6048     base($reg);
 6049     index($ireg);
 6050     scale($scale);
 6051     disp(0x0);
 6052   %}
 6053 %}
 6054 
 6055 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6056 %{
 6057   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6058   constraint(ALLOC_IN_RC(ptr_reg));
 6059   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6060   op_cost(0);
 6061   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6062   interface(MEMORY_INTER) %{
 6063     base($reg);
 6064     index($lreg);
 6065     scale($scale);
 6066     disp(0x0);
 6067   %}
 6068 %}
 6069 
 6070 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6071 %{
 6072   predicate(CompressedOops::shift() == 0);
 6073   constraint(ALLOC_IN_RC(ptr_reg));
 6074   match(AddP (DecodeN reg) (ConvI2L ireg));
 6075   op_cost(0);
 6076   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6077   interface(MEMORY_INTER) %{
 6078     base($reg);
 6079     index($ireg);
 6080     scale(0x0);
 6081     disp(0x0);
 6082   %}
 6083 %}
 6084 
 6085 operand indIndexN(iRegN reg, iRegL lreg)
 6086 %{
 6087   predicate(CompressedOops::shift() == 0);
 6088   constraint(ALLOC_IN_RC(ptr_reg));
 6089   match(AddP (DecodeN reg) lreg);
 6090   op_cost(0);
 6091   format %{ "$reg, $lreg\t# narrow" %}
 6092   interface(MEMORY_INTER) %{
 6093     base($reg);
 6094     index($lreg);
 6095     scale(0x0);
 6096     disp(0x0);
 6097   %}
 6098 %}
 6099 
 6100 operand indOffIN(iRegN reg, immIOffset off)
 6101 %{
 6102   predicate(CompressedOops::shift() == 0);
 6103   constraint(ALLOC_IN_RC(ptr_reg));
 6104   match(AddP (DecodeN reg) off);
 6105   op_cost(0);
 6106   format %{ "[$reg, $off]\t# narrow" %}
 6107   interface(MEMORY_INTER) %{
 6108     base($reg);
 6109     index(0xffffffff);
 6110     scale(0x0);
 6111     disp($off);
 6112   %}
 6113 %}
 6114 
 6115 operand indOffLN(iRegN reg, immLoffset off)
 6116 %{
 6117   predicate(CompressedOops::shift() == 0);
 6118   constraint(ALLOC_IN_RC(ptr_reg));
 6119   match(AddP (DecodeN reg) off);
 6120   op_cost(0);
 6121   format %{ "[$reg, $off]\t# narrow" %}
 6122   interface(MEMORY_INTER) %{
 6123     base($reg);
 6124     index(0xffffffff);
 6125     scale(0x0);
 6126     disp($off);
 6127   %}
 6128 %}
 6129 
 6130 
 6131 
 6132 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6133 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6134 %{
 6135   constraint(ALLOC_IN_RC(ptr_reg));
 6136   match(AddP reg off);
 6137   op_cost(0);
 6138   format %{ "[$reg, $off]" %}
 6139   interface(MEMORY_INTER) %{
 6140     base($reg);
 6141     index(0xffffffff);
 6142     scale(0x0);
 6143     disp($off);
 6144   %}
 6145 %}
 6146 
 6147 //----------Special Memory Operands--------------------------------------------
 6148 // Stack Slot Operand - This operand is used for loading and storing temporary
 6149 //                      values on the stack where a match requires a value to
 6150 //                      flow through memory.
 6151 operand stackSlotP(sRegP reg)
 6152 %{
 6153   constraint(ALLOC_IN_RC(stack_slots));
 6154   op_cost(100);
 6155   // No match rule because this operand is only generated in matching
 6156   // match(RegP);
 6157   format %{ "[$reg]" %}
 6158   interface(MEMORY_INTER) %{
 6159     base(0x1e);  // RSP
 6160     index(0x0);  // No Index
 6161     scale(0x0);  // No Scale
 6162     disp($reg);  // Stack Offset
 6163   %}
 6164 %}
 6165 
 6166 operand stackSlotI(sRegI reg)
 6167 %{
 6168   constraint(ALLOC_IN_RC(stack_slots));
 6169   // No match rule because this operand is only generated in matching
 6170   // match(RegI);
 6171   format %{ "[$reg]" %}
 6172   interface(MEMORY_INTER) %{
 6173     base(0x1e);  // RSP
 6174     index(0x0);  // No Index
 6175     scale(0x0);  // No Scale
 6176     disp($reg);  // Stack Offset
 6177   %}
 6178 %}
 6179 
 6180 operand stackSlotF(sRegF reg)
 6181 %{
 6182   constraint(ALLOC_IN_RC(stack_slots));
 6183   // No match rule because this operand is only generated in matching
 6184   // match(RegF);
 6185   format %{ "[$reg]" %}
 6186   interface(MEMORY_INTER) %{
 6187     base(0x1e);  // RSP
 6188     index(0x0);  // No Index
 6189     scale(0x0);  // No Scale
 6190     disp($reg);  // Stack Offset
 6191   %}
 6192 %}
 6193 
 6194 operand stackSlotD(sRegD reg)
 6195 %{
 6196   constraint(ALLOC_IN_RC(stack_slots));
 6197   // No match rule because this operand is only generated in matching
 6198   // match(RegD);
 6199   format %{ "[$reg]" %}
 6200   interface(MEMORY_INTER) %{
 6201     base(0x1e);  // RSP
 6202     index(0x0);  // No Index
 6203     scale(0x0);  // No Scale
 6204     disp($reg);  // Stack Offset
 6205   %}
 6206 %}
 6207 
 6208 operand stackSlotL(sRegL reg)
 6209 %{
 6210   constraint(ALLOC_IN_RC(stack_slots));
 6211   // No match rule because this operand is only generated in matching
 6212   // match(RegL);
 6213   format %{ "[$reg]" %}
 6214   interface(MEMORY_INTER) %{
 6215     base(0x1e);  // RSP
 6216     index(0x0);  // No Index
 6217     scale(0x0);  // No Scale
 6218     disp($reg);  // Stack Offset
 6219   %}
 6220 %}
 6221 
 6222 // Operands for expressing Control Flow
 6223 // NOTE: Label is a predefined operand which should not be redefined in
 6224 //       the AD file. It is generically handled within the ADLC.
 6225 
 6226 //----------Conditional Branch Operands----------------------------------------
 6227 // Comparison Op  - This is the operation of the comparison, and is limited to
 6228 //                  the following set of codes:
 6229 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6230 //
 6231 // Other attributes of the comparison, such as unsignedness, are specified
 6232 // by the comparison instruction that sets a condition code flags register.
 6233 // That result is represented by a flags operand whose subtype is appropriate
 6234 // to the unsignedness (etc.) of the comparison.
 6235 //
 6236 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6237 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6238 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6239 
 6240 // used for signed integral comparisons and fp comparisons
 6241 
 6242 operand cmpOp()
 6243 %{
 6244   match(Bool);
 6245 
 6246   format %{ "" %}
 6247   interface(COND_INTER) %{
 6248     equal(0x0, "eq");
 6249     not_equal(0x1, "ne");
 6250     less(0xb, "lt");
 6251     greater_equal(0xa, "ge");
 6252     less_equal(0xd, "le");
 6253     greater(0xc, "gt");
 6254     overflow(0x6, "vs");
 6255     no_overflow(0x7, "vc");
 6256   %}
 6257 %}
 6258 
 6259 // used for unsigned integral comparisons
 6260 
 6261 operand cmpOpU()
 6262 %{
 6263   match(Bool);
 6264 
 6265   format %{ "" %}
 6266   interface(COND_INTER) %{
 6267     equal(0x0, "eq");
 6268     not_equal(0x1, "ne");
 6269     less(0x3, "lo");
 6270     greater_equal(0x2, "hs");
 6271     less_equal(0x9, "ls");
 6272     greater(0x8, "hi");
 6273     overflow(0x6, "vs");
 6274     no_overflow(0x7, "vc");
 6275   %}
 6276 %}
 6277 
 6278 // used for certain integral comparisons which can be
 6279 // converted to cbxx or tbxx instructions
 6280 
 6281 operand cmpOpEqNe()
 6282 %{
 6283   match(Bool);
 6284   op_cost(0);
 6285   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6286             || n->as_Bool()->_test._test == BoolTest::eq);
 6287 
 6288   format %{ "" %}
 6289   interface(COND_INTER) %{
 6290     equal(0x0, "eq");
 6291     not_equal(0x1, "ne");
 6292     less(0xb, "lt");
 6293     greater_equal(0xa, "ge");
 6294     less_equal(0xd, "le");
 6295     greater(0xc, "gt");
 6296     overflow(0x6, "vs");
 6297     no_overflow(0x7, "vc");
 6298   %}
 6299 %}
 6300 
 6301 // used for certain integral comparisons which can be
 6302 // converted to cbxx or tbxx instructions
 6303 
 6304 operand cmpOpLtGe()
 6305 %{
 6306   match(Bool);
 6307   op_cost(0);
 6308 
 6309   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6310             || n->as_Bool()->_test._test == BoolTest::ge);
 6311 
 6312   format %{ "" %}
 6313   interface(COND_INTER) %{
 6314     equal(0x0, "eq");
 6315     not_equal(0x1, "ne");
 6316     less(0xb, "lt");
 6317     greater_equal(0xa, "ge");
 6318     less_equal(0xd, "le");
 6319     greater(0xc, "gt");
 6320     overflow(0x6, "vs");
 6321     no_overflow(0x7, "vc");
 6322   %}
 6323 %}
 6324 
 6325 // used for certain unsigned integral comparisons which can be
 6326 // converted to cbxx or tbxx instructions
 6327 
 6328 operand cmpOpUEqNeLtGe()
 6329 %{
 6330   match(Bool);
 6331   op_cost(0);
 6332 
 6333   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6334             || n->as_Bool()->_test._test == BoolTest::ne
 6335             || n->as_Bool()->_test._test == BoolTest::lt
 6336             || n->as_Bool()->_test._test == BoolTest::ge);
 6337 
 6338   format %{ "" %}
 6339   interface(COND_INTER) %{
 6340     equal(0x0, "eq");
 6341     not_equal(0x1, "ne");
 6342     less(0xb, "lt");
 6343     greater_equal(0xa, "ge");
 6344     less_equal(0xd, "le");
 6345     greater(0xc, "gt");
 6346     overflow(0x6, "vs");
 6347     no_overflow(0x7, "vc");
 6348   %}
 6349 %}
 6350 
 6351 // Special operand allowing long args to int ops to be truncated for free
 6352 
 6353 operand iRegL2I(iRegL reg) %{
 6354 
 6355   op_cost(0);
 6356 
 6357   match(ConvL2I reg);
 6358 
 6359   format %{ "l2i($reg)" %}
 6360 
 6361   interface(REG_INTER)
 6362 %}
 6363 
 6364 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6365 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6366 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6367 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6368 
 6369 //----------OPERAND CLASSES----------------------------------------------------
 6370 // Operand Classes are groups of operands that are used as to simplify
 6371 // instruction definitions by not requiring the AD writer to specify
 6372 // separate instructions for every form of operand when the
 6373 // instruction accepts multiple operand types with the same basic
 6374 // encoding and format. The classic case of this is memory operands.
 6375 
 6376 // memory is used to define read/write location for load/store
 6377 // instruction defs. we can turn a memory op into an Address
 6378 
 6379 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6380                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6381 
 6382 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6383                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6384 
 6385 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6386                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6387 
 6388 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6389                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6390 
 6391 // All of the memory operands. For the pipeline description.
 6392 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6393                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6394                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6395 
 6396 
 6397 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6398 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6399 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6400 // can be elided because the 32-bit instruction will just employ the
 6401 // lower 32 bits anyway.
 6402 //
 6403 // n.b. this does not elide all L2I conversions. if the truncated
 6404 // value is consumed by more than one operation then the ConvL2I
 6405 // cannot be bundled into the consuming nodes so an l2i gets planted
 6406 // (actually a movw $dst $src) and the downstream instructions consume
 6407 // the result of the l2i as an iRegI input. That's a shame since the
 6408 // movw is actually redundant but its not too costly.
 6409 
 6410 opclass iRegIorL2I(iRegI, iRegL2I);
 6411 
 6412 //----------PIPELINE-----------------------------------------------------------
 6413 // Rules which define the behavior of the target architectures pipeline.
 6414 
 6415 // For specific pipelines, eg A53, define the stages of that pipeline
 6416 //pipe_desc(ISS, EX1, EX2, WR);
 6417 #define ISS S0
 6418 #define EX1 S1
 6419 #define EX2 S2
 6420 #define WR  S3
 6421 
 6422 // Integer ALU reg operation
 6423 pipeline %{
 6424 
 6425 attributes %{
 6426   // ARM instructions are of fixed length
 6427   fixed_size_instructions;        // Fixed size instructions TODO does
 6428   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6429   // ARM instructions come in 32-bit word units
 6430   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6431   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6432   instruction_fetch_units = 1;       // of 64 bytes
 6433 
 6434   // List of nop instructions
 6435   nops( MachNop );
 6436 %}
 6437 
 6438 // We don't use an actual pipeline model so don't care about resources
 6439 // or description. we do use pipeline classes to introduce fixed
 6440 // latencies
 6441 
 6442 //----------RESOURCES----------------------------------------------------------
 6443 // Resources are the functional units available to the machine
 6444 
 6445 resources( INS0, INS1, INS01 = INS0 | INS1,
 6446            ALU0, ALU1, ALU = ALU0 | ALU1,
 6447            MAC,
 6448            DIV,
 6449            BRANCH,
 6450            LDST,
 6451            NEON_FP);
 6452 
 6453 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6454 // Pipeline Description specifies the stages in the machine's pipeline
 6455 
 6456 // Define the pipeline as a generic 6 stage pipeline
 6457 pipe_desc(S0, S1, S2, S3, S4, S5);
 6458 
 6459 //----------PIPELINE CLASSES---------------------------------------------------
 6460 // Pipeline Classes describe the stages in which input and output are
 6461 // referenced by the hardware pipeline.
 6462 
 6463 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6464 %{
 6465   single_instruction;
 6466   src1   : S1(read);
 6467   src2   : S2(read);
 6468   dst    : S5(write);
 6469   INS01  : ISS;
 6470   NEON_FP : S5;
 6471 %}
 6472 
 6473 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6474 %{
 6475   single_instruction;
 6476   src1   : S1(read);
 6477   src2   : S2(read);
 6478   dst    : S5(write);
 6479   INS01  : ISS;
 6480   NEON_FP : S5;
 6481 %}
 6482 
 6483 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6484 %{
 6485   single_instruction;
 6486   src    : S1(read);
 6487   dst    : S5(write);
 6488   INS01  : ISS;
 6489   NEON_FP : S5;
 6490 %}
 6491 
 6492 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6493 %{
 6494   single_instruction;
 6495   src    : S1(read);
 6496   dst    : S5(write);
 6497   INS01  : ISS;
 6498   NEON_FP : S5;
 6499 %}
 6500 
 6501 pipe_class fp_d2f(vRegF dst, vRegD src)
 6502 %{
 6503   single_instruction;
 6504   src    : S1(read);
 6505   dst    : S5(write);
 6506   INS01  : ISS;
 6507   NEON_FP : S5;
 6508 %}
 6509 
 6510 pipe_class fp_f2d(vRegD dst, vRegF src)
 6511 %{
 6512   single_instruction;
 6513   src    : S1(read);
 6514   dst    : S5(write);
 6515   INS01  : ISS;
 6516   NEON_FP : S5;
 6517 %}
 6518 
 6519 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6520 %{
 6521   single_instruction;
 6522   src    : S1(read);
 6523   dst    : S5(write);
 6524   INS01  : ISS;
 6525   NEON_FP : S5;
 6526 %}
 6527 
 6528 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6529 %{
 6530   single_instruction;
 6531   src    : S1(read);
 6532   dst    : S5(write);
 6533   INS01  : ISS;
 6534   NEON_FP : S5;
 6535 %}
 6536 
 6537 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6538 %{
 6539   single_instruction;
 6540   src    : S1(read);
 6541   dst    : S5(write);
 6542   INS01  : ISS;
 6543   NEON_FP : S5;
 6544 %}
 6545 
 6546 pipe_class fp_l2f(vRegF dst, iRegL src)
 6547 %{
 6548   single_instruction;
 6549   src    : S1(read);
 6550   dst    : S5(write);
 6551   INS01  : ISS;
 6552   NEON_FP : S5;
 6553 %}
 6554 
 6555 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6556 %{
 6557   single_instruction;
 6558   src    : S1(read);
 6559   dst    : S5(write);
 6560   INS01  : ISS;
 6561   NEON_FP : S5;
 6562 %}
 6563 
 6564 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6565 %{
 6566   single_instruction;
 6567   src    : S1(read);
 6568   dst    : S5(write);
 6569   INS01  : ISS;
 6570   NEON_FP : S5;
 6571 %}
 6572 
 6573 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6574 %{
 6575   single_instruction;
 6576   src    : S1(read);
 6577   dst    : S5(write);
 6578   INS01  : ISS;
 6579   NEON_FP : S5;
 6580 %}
 6581 
 6582 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6583 %{
 6584   single_instruction;
 6585   src    : S1(read);
 6586   dst    : S5(write);
 6587   INS01  : ISS;
 6588   NEON_FP : S5;
 6589 %}
 6590 
 6591 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6592 %{
 6593   single_instruction;
 6594   src1   : S1(read);
 6595   src2   : S2(read);
 6596   dst    : S5(write);
 6597   INS0   : ISS;
 6598   NEON_FP : S5;
 6599 %}
 6600 
 6601 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6602 %{
 6603   single_instruction;
 6604   src1   : S1(read);
 6605   src2   : S2(read);
 6606   dst    : S5(write);
 6607   INS0   : ISS;
 6608   NEON_FP : S5;
 6609 %}
 6610 
 6611 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6612 %{
 6613   single_instruction;
 6614   cr     : S1(read);
 6615   src1   : S1(read);
 6616   src2   : S1(read);
 6617   dst    : S3(write);
 6618   INS01  : ISS;
 6619   NEON_FP : S3;
 6620 %}
 6621 
 6622 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6623 %{
 6624   single_instruction;
 6625   cr     : S1(read);
 6626   src1   : S1(read);
 6627   src2   : S1(read);
 6628   dst    : S3(write);
 6629   INS01  : ISS;
 6630   NEON_FP : S3;
 6631 %}
 6632 
 6633 pipe_class fp_imm_s(vRegF dst)
 6634 %{
 6635   single_instruction;
 6636   dst    : S3(write);
 6637   INS01  : ISS;
 6638   NEON_FP : S3;
 6639 %}
 6640 
 6641 pipe_class fp_imm_d(vRegD dst)
 6642 %{
 6643   single_instruction;
 6644   dst    : S3(write);
 6645   INS01  : ISS;
 6646   NEON_FP : S3;
 6647 %}
 6648 
 6649 pipe_class fp_load_constant_s(vRegF dst)
 6650 %{
 6651   single_instruction;
 6652   dst    : S4(write);
 6653   INS01  : ISS;
 6654   NEON_FP : S4;
 6655 %}
 6656 
 6657 pipe_class fp_load_constant_d(vRegD dst)
 6658 %{
 6659   single_instruction;
 6660   dst    : S4(write);
 6661   INS01  : ISS;
 6662   NEON_FP : S4;
 6663 %}
 6664 
 6665 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6666 %{
 6667   single_instruction;
 6668   dst    : S5(write);
 6669   src1   : S1(read);
 6670   src2   : S1(read);
 6671   INS01  : ISS;
 6672   NEON_FP : S5;
 6673 %}
 6674 
 6675 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6676 %{
 6677   single_instruction;
 6678   dst    : S5(write);
 6679   src1   : S1(read);
 6680   src2   : S1(read);
 6681   INS0   : ISS;
 6682   NEON_FP : S5;
 6683 %}
 6684 
 6685 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6686 %{
 6687   single_instruction;
 6688   dst    : S5(write);
 6689   src1   : S1(read);
 6690   src2   : S1(read);
 6691   dst    : S1(read);
 6692   INS01  : ISS;
 6693   NEON_FP : S5;
 6694 %}
 6695 
 6696 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6697 %{
 6698   single_instruction;
 6699   dst    : S5(write);
 6700   src1   : S1(read);
 6701   src2   : S1(read);
 6702   dst    : S1(read);
 6703   INS0   : ISS;
 6704   NEON_FP : S5;
 6705 %}
 6706 
 6707 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6708 %{
 6709   single_instruction;
 6710   dst    : S4(write);
 6711   src1   : S2(read);
 6712   src2   : S2(read);
 6713   INS01  : ISS;
 6714   NEON_FP : S4;
 6715 %}
 6716 
 6717 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6718 %{
 6719   single_instruction;
 6720   dst    : S4(write);
 6721   src1   : S2(read);
 6722   src2   : S2(read);
 6723   INS0   : ISS;
 6724   NEON_FP : S4;
 6725 %}
 6726 
 6727 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6728 %{
 6729   single_instruction;
 6730   dst    : S3(write);
 6731   src1   : S2(read);
 6732   src2   : S2(read);
 6733   INS01  : ISS;
 6734   NEON_FP : S3;
 6735 %}
 6736 
 6737 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6738 %{
 6739   single_instruction;
 6740   dst    : S3(write);
 6741   src1   : S2(read);
 6742   src2   : S2(read);
 6743   INS0   : ISS;
 6744   NEON_FP : S3;
 6745 %}
 6746 
 6747 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6748 %{
 6749   single_instruction;
 6750   dst    : S3(write);
 6751   src    : S1(read);
 6752   shift  : S1(read);
 6753   INS01  : ISS;
 6754   NEON_FP : S3;
 6755 %}
 6756 
 6757 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6758 %{
 6759   single_instruction;
 6760   dst    : S3(write);
 6761   src    : S1(read);
 6762   shift  : S1(read);
 6763   INS0   : ISS;
 6764   NEON_FP : S3;
 6765 %}
 6766 
 6767 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6768 %{
 6769   single_instruction;
 6770   dst    : S3(write);
 6771   src    : S1(read);
 6772   INS01  : ISS;
 6773   NEON_FP : S3;
 6774 %}
 6775 
 6776 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6777 %{
 6778   single_instruction;
 6779   dst    : S3(write);
 6780   src    : S1(read);
 6781   INS0   : ISS;
 6782   NEON_FP : S3;
 6783 %}
 6784 
 6785 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6786 %{
 6787   single_instruction;
 6788   dst    : S5(write);
 6789   src1   : S1(read);
 6790   src2   : S1(read);
 6791   INS01  : ISS;
 6792   NEON_FP : S5;
 6793 %}
 6794 
 6795 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6796 %{
 6797   single_instruction;
 6798   dst    : S5(write);
 6799   src1   : S1(read);
 6800   src2   : S1(read);
 6801   INS0   : ISS;
 6802   NEON_FP : S5;
 6803 %}
 6804 
 6805 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6806 %{
 6807   single_instruction;
 6808   dst    : S5(write);
 6809   src1   : S1(read);
 6810   src2   : S1(read);
 6811   INS0   : ISS;
 6812   NEON_FP : S5;
 6813 %}
 6814 
 6815 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6816 %{
 6817   single_instruction;
 6818   dst    : S5(write);
 6819   src1   : S1(read);
 6820   src2   : S1(read);
 6821   INS0   : ISS;
 6822   NEON_FP : S5;
 6823 %}
 6824 
 6825 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6826 %{
 6827   single_instruction;
 6828   dst    : S5(write);
 6829   src    : S1(read);
 6830   INS0   : ISS;
 6831   NEON_FP : S5;
 6832 %}
 6833 
 6834 pipe_class vunop_fp64(vecD dst, vecD src)
 6835 %{
 6836   single_instruction;
 6837   dst    : S5(write);
 6838   src    : S1(read);
 6839   INS01  : ISS;
 6840   NEON_FP : S5;
 6841 %}
 6842 
 6843 pipe_class vunop_fp128(vecX dst, vecX src)
 6844 %{
 6845   single_instruction;
 6846   dst    : S5(write);
 6847   src    : S1(read);
 6848   INS0   : ISS;
 6849   NEON_FP : S5;
 6850 %}
 6851 
 6852 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6853 %{
 6854   single_instruction;
 6855   dst    : S3(write);
 6856   src    : S1(read);
 6857   INS01  : ISS;
 6858   NEON_FP : S3;
 6859 %}
 6860 
 6861 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6862 %{
 6863   single_instruction;
 6864   dst    : S3(write);
 6865   src    : S1(read);
 6866   INS01  : ISS;
 6867   NEON_FP : S3;
 6868 %}
 6869 
 6870 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6871 %{
 6872   single_instruction;
 6873   dst    : S3(write);
 6874   src    : S1(read);
 6875   INS01  : ISS;
 6876   NEON_FP : S3;
 6877 %}
 6878 
 6879 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6880 %{
 6881   single_instruction;
 6882   dst    : S3(write);
 6883   src    : S1(read);
 6884   INS01  : ISS;
 6885   NEON_FP : S3;
 6886 %}
 6887 
 6888 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6889 %{
 6890   single_instruction;
 6891   dst    : S3(write);
 6892   src    : S1(read);
 6893   INS01  : ISS;
 6894   NEON_FP : S3;
 6895 %}
 6896 
 6897 pipe_class vmovi_reg_imm64(vecD dst)
 6898 %{
 6899   single_instruction;
 6900   dst    : S3(write);
 6901   INS01  : ISS;
 6902   NEON_FP : S3;
 6903 %}
 6904 
 6905 pipe_class vmovi_reg_imm128(vecX dst)
 6906 %{
 6907   single_instruction;
 6908   dst    : S3(write);
 6909   INS0   : ISS;
 6910   NEON_FP : S3;
 6911 %}
 6912 
 6913 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6914 %{
 6915   single_instruction;
 6916   dst    : S5(write);
 6917   mem    : ISS(read);
 6918   INS01  : ISS;
 6919   NEON_FP : S3;
 6920 %}
 6921 
 6922 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6923 %{
 6924   single_instruction;
 6925   dst    : S5(write);
 6926   mem    : ISS(read);
 6927   INS01  : ISS;
 6928   NEON_FP : S3;
 6929 %}
 6930 
 6931 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6932 %{
 6933   single_instruction;
 6934   mem    : ISS(read);
 6935   src    : S2(read);
 6936   INS01  : ISS;
 6937   NEON_FP : S3;
 6938 %}
 6939 
 6940 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6941 %{
 6942   single_instruction;
 6943   mem    : ISS(read);
 6944   src    : S2(read);
 6945   INS01  : ISS;
 6946   NEON_FP : S3;
 6947 %}
 6948 
 6949 //------- Integer ALU operations --------------------------
 6950 
 6951 // Integer ALU reg-reg operation
 6952 // Operands needed in EX1, result generated in EX2
 6953 // Eg.  ADD     x0, x1, x2
 6954 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6955 %{
 6956   single_instruction;
 6957   dst    : EX2(write);
 6958   src1   : EX1(read);
 6959   src2   : EX1(read);
 6960   INS01  : ISS; // Dual issue as instruction 0 or 1
 6961   ALU    : EX2;
 6962 %}
 6963 
 6964 // Integer ALU reg-reg operation with constant shift
 6965 // Shifted register must be available in LATE_ISS instead of EX1
 6966 // Eg.  ADD     x0, x1, x2, LSL #2
 6967 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6968 %{
 6969   single_instruction;
 6970   dst    : EX2(write);
 6971   src1   : EX1(read);
 6972   src2   : ISS(read);
 6973   INS01  : ISS;
 6974   ALU    : EX2;
 6975 %}
 6976 
 6977 // Integer ALU reg operation with constant shift
 6978 // Eg.  LSL     x0, x1, #shift
 6979 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6980 %{
 6981   single_instruction;
 6982   dst    : EX2(write);
 6983   src1   : ISS(read);
 6984   INS01  : ISS;
 6985   ALU    : EX2;
 6986 %}
 6987 
 6988 // Integer ALU reg-reg operation with variable shift
 6989 // Both operands must be available in LATE_ISS instead of EX1
 6990 // Result is available in EX1 instead of EX2
 6991 // Eg.  LSLV    x0, x1, x2
 6992 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6993 %{
 6994   single_instruction;
 6995   dst    : EX1(write);
 6996   src1   : ISS(read);
 6997   src2   : ISS(read);
 6998   INS01  : ISS;
 6999   ALU    : EX1;
 7000 %}
 7001 
 7002 // Integer ALU reg-reg operation with extract
 7003 // As for _vshift above, but result generated in EX2
 7004 // Eg.  EXTR    x0, x1, x2, #N
 7005 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7006 %{
 7007   single_instruction;
 7008   dst    : EX2(write);
 7009   src1   : ISS(read);
 7010   src2   : ISS(read);
 7011   INS1   : ISS; // Can only dual issue as Instruction 1
 7012   ALU    : EX1;
 7013 %}
 7014 
 7015 // Integer ALU reg operation
 7016 // Eg.  NEG     x0, x1
 7017 pipe_class ialu_reg(iRegI dst, iRegI src)
 7018 %{
 7019   single_instruction;
 7020   dst    : EX2(write);
 7021   src    : EX1(read);
 7022   INS01  : ISS;
 7023   ALU    : EX2;
 7024 %}
 7025 
 7026 // Integer ALU reg mmediate operation
 7027 // Eg.  ADD     x0, x1, #N
 7028 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7029 %{
 7030   single_instruction;
 7031   dst    : EX2(write);
 7032   src1   : EX1(read);
 7033   INS01  : ISS;
 7034   ALU    : EX2;
 7035 %}
 7036 
 7037 // Integer ALU immediate operation (no source operands)
 7038 // Eg.  MOV     x0, #N
 7039 pipe_class ialu_imm(iRegI dst)
 7040 %{
 7041   single_instruction;
 7042   dst    : EX1(write);
 7043   INS01  : ISS;
 7044   ALU    : EX1;
 7045 %}
 7046 
 7047 //------- Compare operation -------------------------------
 7048 
 7049 // Compare reg-reg
 7050 // Eg.  CMP     x0, x1
 7051 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7052 %{
 7053   single_instruction;
 7054 //  fixed_latency(16);
 7055   cr     : EX2(write);
 7056   op1    : EX1(read);
 7057   op2    : EX1(read);
 7058   INS01  : ISS;
 7059   ALU    : EX2;
 7060 %}
 7061 
 7062 // Compare reg-reg
 7063 // Eg.  CMP     x0, #N
 7064 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7065 %{
 7066   single_instruction;
 7067 //  fixed_latency(16);
 7068   cr     : EX2(write);
 7069   op1    : EX1(read);
 7070   INS01  : ISS;
 7071   ALU    : EX2;
 7072 %}
 7073 
 7074 //------- Conditional instructions ------------------------
 7075 
 7076 // Conditional no operands
 7077 // Eg.  CSINC   x0, zr, zr, <cond>
 7078 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7079 %{
 7080   single_instruction;
 7081   cr     : EX1(read);
 7082   dst    : EX2(write);
 7083   INS01  : ISS;
 7084   ALU    : EX2;
 7085 %}
 7086 
 7087 // Conditional 2 operand
 7088 // EG.  CSEL    X0, X1, X2, <cond>
 7089 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7090 %{
 7091   single_instruction;
 7092   cr     : EX1(read);
 7093   src1   : EX1(read);
 7094   src2   : EX1(read);
 7095   dst    : EX2(write);
 7096   INS01  : ISS;
 7097   ALU    : EX2;
 7098 %}
 7099 
 7100 // Conditional 2 operand
 7101 // EG.  CSEL    X0, X1, X2, <cond>
 7102 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7103 %{
 7104   single_instruction;
 7105   cr     : EX1(read);
 7106   src    : EX1(read);
 7107   dst    : EX2(write);
 7108   INS01  : ISS;
 7109   ALU    : EX2;
 7110 %}
 7111 
 7112 //------- Multiply pipeline operations --------------------
 7113 
 7114 // Multiply reg-reg
 7115 // Eg.  MUL     w0, w1, w2
 7116 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7117 %{
 7118   single_instruction;
 7119   dst    : WR(write);
 7120   src1   : ISS(read);
 7121   src2   : ISS(read);
 7122   INS01  : ISS;
 7123   MAC    : WR;
 7124 %}
 7125 
 7126 // Multiply accumulate
 7127 // Eg.  MADD    w0, w1, w2, w3
 7128 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7129 %{
 7130   single_instruction;
 7131   dst    : WR(write);
 7132   src1   : ISS(read);
 7133   src2   : ISS(read);
 7134   src3   : ISS(read);
 7135   INS01  : ISS;
 7136   MAC    : WR;
 7137 %}
 7138 
 7139 // Eg.  MUL     w0, w1, w2
 7140 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7141 %{
 7142   single_instruction;
 7143   fixed_latency(3); // Maximum latency for 64 bit mul
 7144   dst    : WR(write);
 7145   src1   : ISS(read);
 7146   src2   : ISS(read);
 7147   INS01  : ISS;
 7148   MAC    : WR;
 7149 %}
 7150 
 7151 // Multiply accumulate
 7152 // Eg.  MADD    w0, w1, w2, w3
 7153 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7154 %{
 7155   single_instruction;
 7156   fixed_latency(3); // Maximum latency for 64 bit mul
 7157   dst    : WR(write);
 7158   src1   : ISS(read);
 7159   src2   : ISS(read);
 7160   src3   : ISS(read);
 7161   INS01  : ISS;
 7162   MAC    : WR;
 7163 %}
 7164 
 7165 //------- Divide pipeline operations --------------------
 7166 
 7167 // Eg.  SDIV    w0, w1, w2
 7168 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7169 %{
 7170   single_instruction;
 7171   fixed_latency(8); // Maximum latency for 32 bit divide
 7172   dst    : WR(write);
 7173   src1   : ISS(read);
 7174   src2   : ISS(read);
 7175   INS0   : ISS; // Can only dual issue as instruction 0
 7176   DIV    : WR;
 7177 %}
 7178 
 7179 // Eg.  SDIV    x0, x1, x2
 7180 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7181 %{
 7182   single_instruction;
 7183   fixed_latency(16); // Maximum latency for 64 bit divide
 7184   dst    : WR(write);
 7185   src1   : ISS(read);
 7186   src2   : ISS(read);
 7187   INS0   : ISS; // Can only dual issue as instruction 0
 7188   DIV    : WR;
 7189 %}
 7190 
 7191 //------- Load pipeline operations ------------------------
 7192 
 7193 // Load - prefetch
 7194 // Eg.  PFRM    <mem>
 7195 pipe_class iload_prefetch(memory mem)
 7196 %{
 7197   single_instruction;
 7198   mem    : ISS(read);
 7199   INS01  : ISS;
 7200   LDST   : WR;
 7201 %}
 7202 
 7203 // Load - reg, mem
 7204 // Eg.  LDR     x0, <mem>
 7205 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7206 %{
 7207   single_instruction;
 7208   dst    : WR(write);
 7209   mem    : ISS(read);
 7210   INS01  : ISS;
 7211   LDST   : WR;
 7212 %}
 7213 
 7214 // Load - reg, reg
 7215 // Eg.  LDR     x0, [sp, x1]
 7216 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7217 %{
 7218   single_instruction;
 7219   dst    : WR(write);
 7220   src    : ISS(read);
 7221   INS01  : ISS;
 7222   LDST   : WR;
 7223 %}
 7224 
 7225 //------- Store pipeline operations -----------------------
 7226 
 7227 // Store - zr, mem
 7228 // Eg.  STR     zr, <mem>
 7229 pipe_class istore_mem(memory mem)
 7230 %{
 7231   single_instruction;
 7232   mem    : ISS(read);
 7233   INS01  : ISS;
 7234   LDST   : WR;
 7235 %}
 7236 
 7237 // Store - reg, mem
 7238 // Eg.  STR     x0, <mem>
 7239 pipe_class istore_reg_mem(iRegI src, memory mem)
 7240 %{
 7241   single_instruction;
 7242   mem    : ISS(read);
 7243   src    : EX2(read);
 7244   INS01  : ISS;
 7245   LDST   : WR;
 7246 %}
 7247 
 7248 // Store - reg, reg
 7249 // Eg. STR      x0, [sp, x1]
 7250 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7251 %{
 7252   single_instruction;
 7253   dst    : ISS(read);
 7254   src    : EX2(read);
 7255   INS01  : ISS;
 7256   LDST   : WR;
 7257 %}
 7258 
 7259 //------- Store pipeline operations -----------------------
 7260 
 7261 // Branch
 7262 pipe_class pipe_branch()
 7263 %{
 7264   single_instruction;
 7265   INS01  : ISS;
 7266   BRANCH : EX1;
 7267 %}
 7268 
 7269 // Conditional branch
 7270 pipe_class pipe_branch_cond(rFlagsReg cr)
 7271 %{
 7272   single_instruction;
 7273   cr     : EX1(read);
 7274   INS01  : ISS;
 7275   BRANCH : EX1;
 7276 %}
 7277 
 7278 // Compare & Branch
 7279 // EG.  CBZ/CBNZ
 7280 pipe_class pipe_cmp_branch(iRegI op1)
 7281 %{
 7282   single_instruction;
 7283   op1    : EX1(read);
 7284   INS01  : ISS;
 7285   BRANCH : EX1;
 7286 %}
 7287 
 7288 //------- Synchronisation operations ----------------------
 7289 
 7290 // Any operation requiring serialization.
 7291 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7292 pipe_class pipe_serial()
 7293 %{
 7294   single_instruction;
 7295   force_serialization;
 7296   fixed_latency(16);
 7297   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7298   LDST   : WR;
 7299 %}
 7300 
 7301 // Generic big/slow expanded idiom - also serialized
 7302 pipe_class pipe_slow()
 7303 %{
 7304   instruction_count(10);
 7305   multiple_bundles;
 7306   force_serialization;
 7307   fixed_latency(16);
 7308   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7309   LDST   : WR;
 7310 %}
 7311 
 7312 // Empty pipeline class
 7313 pipe_class pipe_class_empty()
 7314 %{
 7315   single_instruction;
 7316   fixed_latency(0);
 7317 %}
 7318 
 7319 // Default pipeline class.
 7320 pipe_class pipe_class_default()
 7321 %{
 7322   single_instruction;
 7323   fixed_latency(2);
 7324 %}
 7325 
 7326 // Pipeline class for compares.
 7327 pipe_class pipe_class_compare()
 7328 %{
 7329   single_instruction;
 7330   fixed_latency(16);
 7331 %}
 7332 
 7333 // Pipeline class for memory operations.
 7334 pipe_class pipe_class_memory()
 7335 %{
 7336   single_instruction;
 7337   fixed_latency(16);
 7338 %}
 7339 
 7340 // Pipeline class for call.
 7341 pipe_class pipe_class_call()
 7342 %{
 7343   single_instruction;
 7344   fixed_latency(100);
 7345 %}
 7346 
 7347 // Define the class for the Nop node.
 7348 define %{
 7349    MachNop = pipe_class_empty;
 7350 %}
 7351 
 7352 %}
 7353 //----------INSTRUCTIONS-------------------------------------------------------
 7354 //
 7355 // match      -- States which machine-independent subtree may be replaced
 7356 //               by this instruction.
 7357 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7358 //               selection to identify a minimum cost tree of machine
 7359 //               instructions that matches a tree of machine-independent
 7360 //               instructions.
 7361 // format     -- A string providing the disassembly for this instruction.
 7362 //               The value of an instruction's operand may be inserted
 7363 //               by referring to it with a '$' prefix.
 7364 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7365 //               to within an encode class as $primary, $secondary, and $tertiary
 7366 //               rrspectively.  The primary opcode is commonly used to
 7367 //               indicate the type of machine instruction, while secondary
 7368 //               and tertiary are often used for prefix options or addressing
 7369 //               modes.
 7370 // ins_encode -- A list of encode classes with parameters. The encode class
 7371 //               name must have been defined in an 'enc_class' specification
 7372 //               in the encode section of the architecture description.
 7373 
 7374 // ============================================================================
 7375 // Memory (Load/Store) Instructions
 7376 
 7377 // Load Instructions
 7378 
 7379 // Load Byte (8 bit signed)
 7380 instruct loadB(iRegINoSp dst, memory1 mem)
 7381 %{
 7382   match(Set dst (LoadB mem));
 7383   predicate(!needs_acquiring_load(n));
 7384 
 7385   ins_cost(4 * INSN_COST);
 7386   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7387 
 7388   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7389 
 7390   ins_pipe(iload_reg_mem);
 7391 %}
 7392 
 7393 // Load Byte (8 bit signed) into long
 7394 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7395 %{
 7396   match(Set dst (ConvI2L (LoadB mem)));
 7397   predicate(!needs_acquiring_load(n->in(1)));
 7398 
 7399   ins_cost(4 * INSN_COST);
 7400   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7401 
 7402   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7403 
 7404   ins_pipe(iload_reg_mem);
 7405 %}
 7406 
 7407 // Load Byte (8 bit unsigned)
 7408 instruct loadUB(iRegINoSp dst, memory1 mem)
 7409 %{
 7410   match(Set dst (LoadUB mem));
 7411   predicate(!needs_acquiring_load(n));
 7412 
 7413   ins_cost(4 * INSN_COST);
 7414   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7415 
 7416   ins_encode(aarch64_enc_ldrb(dst, mem));
 7417 
 7418   ins_pipe(iload_reg_mem);
 7419 %}
 7420 
 7421 // Load Byte (8 bit unsigned) into long
 7422 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7423 %{
 7424   match(Set dst (ConvI2L (LoadUB mem)));
 7425   predicate(!needs_acquiring_load(n->in(1)));
 7426 
 7427   ins_cost(4 * INSN_COST);
 7428   format %{ "ldrb  $dst, $mem\t# byte" %}
 7429 
 7430   ins_encode(aarch64_enc_ldrb(dst, mem));
 7431 
 7432   ins_pipe(iload_reg_mem);
 7433 %}
 7434 
 7435 // Load Short (16 bit signed)
 7436 instruct loadS(iRegINoSp dst, memory2 mem)
 7437 %{
 7438   match(Set dst (LoadS mem));
 7439   predicate(!needs_acquiring_load(n));
 7440 
 7441   ins_cost(4 * INSN_COST);
 7442   format %{ "ldrshw  $dst, $mem\t# short" %}
 7443 
 7444   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7445 
 7446   ins_pipe(iload_reg_mem);
 7447 %}
 7448 
 7449 // Load Short (16 bit signed) into long
 7450 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7451 %{
 7452   match(Set dst (ConvI2L (LoadS mem)));
 7453   predicate(!needs_acquiring_load(n->in(1)));
 7454 
 7455   ins_cost(4 * INSN_COST);
 7456   format %{ "ldrsh  $dst, $mem\t# short" %}
 7457 
 7458   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7459 
 7460   ins_pipe(iload_reg_mem);
 7461 %}
 7462 
 7463 // Load Char (16 bit unsigned)
 7464 instruct loadUS(iRegINoSp dst, memory2 mem)
 7465 %{
 7466   match(Set dst (LoadUS mem));
 7467   predicate(!needs_acquiring_load(n));
 7468 
 7469   ins_cost(4 * INSN_COST);
 7470   format %{ "ldrh  $dst, $mem\t# short" %}
 7471 
 7472   ins_encode(aarch64_enc_ldrh(dst, mem));
 7473 
 7474   ins_pipe(iload_reg_mem);
 7475 %}
 7476 
 7477 // Load Short/Char (16 bit unsigned) into long
 7478 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7479 %{
 7480   match(Set dst (ConvI2L (LoadUS mem)));
 7481   predicate(!needs_acquiring_load(n->in(1)));
 7482 
 7483   ins_cost(4 * INSN_COST);
 7484   format %{ "ldrh  $dst, $mem\t# short" %}
 7485 
 7486   ins_encode(aarch64_enc_ldrh(dst, mem));
 7487 
 7488   ins_pipe(iload_reg_mem);
 7489 %}
 7490 
 7491 // Load Integer (32 bit signed)
 7492 instruct loadI(iRegINoSp dst, memory4 mem)
 7493 %{
 7494   match(Set dst (LoadI mem));
 7495   predicate(!needs_acquiring_load(n));
 7496 
 7497   ins_cost(4 * INSN_COST);
 7498   format %{ "ldrw  $dst, $mem\t# int" %}
 7499 
 7500   ins_encode(aarch64_enc_ldrw(dst, mem));
 7501 
 7502   ins_pipe(iload_reg_mem);
 7503 %}
 7504 
 7505 // Load Integer (32 bit signed) into long
 7506 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7507 %{
 7508   match(Set dst (ConvI2L (LoadI mem)));
 7509   predicate(!needs_acquiring_load(n->in(1)));
 7510 
 7511   ins_cost(4 * INSN_COST);
 7512   format %{ "ldrsw  $dst, $mem\t# int" %}
 7513 
 7514   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7515 
 7516   ins_pipe(iload_reg_mem);
 7517 %}
 7518 
 7519 // Load Integer (32 bit unsigned) into long
 7520 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7521 %{
 7522   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7523   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7524 
 7525   ins_cost(4 * INSN_COST);
 7526   format %{ "ldrw  $dst, $mem\t# int" %}
 7527 
 7528   ins_encode(aarch64_enc_ldrw(dst, mem));
 7529 
 7530   ins_pipe(iload_reg_mem);
 7531 %}
 7532 
 7533 // Load Long (64 bit signed)
 7534 instruct loadL(iRegLNoSp dst, memory8 mem)
 7535 %{
 7536   match(Set dst (LoadL mem));
 7537   predicate(!needs_acquiring_load(n));
 7538 
 7539   ins_cost(4 * INSN_COST);
 7540   format %{ "ldr  $dst, $mem\t# int" %}
 7541 
 7542   ins_encode(aarch64_enc_ldr(dst, mem));
 7543 
 7544   ins_pipe(iload_reg_mem);
 7545 %}
 7546 
 7547 // Load Range
 7548 instruct loadRange(iRegINoSp dst, memory4 mem)
 7549 %{
 7550   match(Set dst (LoadRange mem));
 7551 
 7552   ins_cost(4 * INSN_COST);
 7553   format %{ "ldrw  $dst, $mem\t# range" %}
 7554 
 7555   ins_encode(aarch64_enc_ldrw(dst, mem));
 7556 
 7557   ins_pipe(iload_reg_mem);
 7558 %}
 7559 
 7560 // Load Pointer
 7561 instruct loadP(iRegPNoSp dst, memory8 mem)
 7562 %{
 7563   match(Set dst (LoadP mem));
 7564   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7565 
 7566   ins_cost(4 * INSN_COST);
 7567   format %{ "ldr  $dst, $mem\t# ptr" %}
 7568 
 7569   ins_encode(aarch64_enc_ldr(dst, mem));
 7570 
 7571   ins_pipe(iload_reg_mem);
 7572 %}
 7573 
 7574 // Load Compressed Pointer
 7575 instruct loadN(iRegNNoSp dst, memory4 mem)
 7576 %{
 7577   match(Set dst (LoadN mem));
 7578   predicate(!needs_acquiring_load(n));
 7579 
 7580   ins_cost(4 * INSN_COST);
 7581   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7582 
 7583   ins_encode(aarch64_enc_ldrw(dst, mem));
 7584 
 7585   ins_pipe(iload_reg_mem);
 7586 %}
 7587 
 7588 // Load Klass Pointer
 7589 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7590 %{
 7591   match(Set dst (LoadKlass mem));
 7592   predicate(!needs_acquiring_load(n));
 7593 
 7594   ins_cost(4 * INSN_COST);
 7595   format %{ "ldr  $dst, $mem\t# class" %}
 7596 
 7597   ins_encode(aarch64_enc_ldr(dst, mem));
 7598 
 7599   ins_pipe(iload_reg_mem);
 7600 %}
 7601 
 7602 // Load Narrow Klass Pointer
 7603 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7604 %{
 7605   match(Set dst (LoadNKlass mem));
 7606   predicate(!needs_acquiring_load(n));
 7607 
 7608   ins_cost(4 * INSN_COST);
 7609   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7610 
 7611   ins_encode(aarch64_enc_ldrw(dst, mem));
 7612 
 7613   ins_pipe(iload_reg_mem);
 7614 %}
 7615 
 7616 // Load Float
 7617 instruct loadF(vRegF dst, memory4 mem)
 7618 %{
 7619   match(Set dst (LoadF mem));
 7620   predicate(!needs_acquiring_load(n));
 7621 
 7622   ins_cost(4 * INSN_COST);
 7623   format %{ "ldrs  $dst, $mem\t# float" %}
 7624 
 7625   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7626 
 7627   ins_pipe(pipe_class_memory);
 7628 %}
 7629 
 7630 // Load Double
 7631 instruct loadD(vRegD dst, memory8 mem)
 7632 %{
 7633   match(Set dst (LoadD mem));
 7634   predicate(!needs_acquiring_load(n));
 7635 
 7636   ins_cost(4 * INSN_COST);
 7637   format %{ "ldrd  $dst, $mem\t# double" %}
 7638 
 7639   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7640 
 7641   ins_pipe(pipe_class_memory);
 7642 %}
 7643 
 7644 
 7645 // Load Int Constant
 7646 instruct loadConI(iRegINoSp dst, immI src)
 7647 %{
 7648   match(Set dst src);
 7649 
 7650   ins_cost(INSN_COST);
 7651   format %{ "mov $dst, $src\t# int" %}
 7652 
 7653   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7654 
 7655   ins_pipe(ialu_imm);
 7656 %}
 7657 
 7658 // Load Long Constant
 7659 instruct loadConL(iRegLNoSp dst, immL src)
 7660 %{
 7661   match(Set dst src);
 7662 
 7663   ins_cost(INSN_COST);
 7664   format %{ "mov $dst, $src\t# long" %}
 7665 
 7666   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7667 
 7668   ins_pipe(ialu_imm);
 7669 %}
 7670 
 7671 // Load Pointer Constant
 7672 
 7673 instruct loadConP(iRegPNoSp dst, immP con)
 7674 %{
 7675   match(Set dst con);
 7676 
 7677   ins_cost(INSN_COST * 4);
 7678   format %{
 7679     "mov  $dst, $con\t# ptr\n\t"
 7680   %}
 7681 
 7682   ins_encode(aarch64_enc_mov_p(dst, con));
 7683 
 7684   ins_pipe(ialu_imm);
 7685 %}
 7686 
 7687 // Load Null Pointer Constant
 7688 
 7689 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7690 %{
 7691   match(Set dst con);
 7692 
 7693   ins_cost(INSN_COST);
 7694   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7695 
 7696   ins_encode(aarch64_enc_mov_p0(dst, con));
 7697 
 7698   ins_pipe(ialu_imm);
 7699 %}
 7700 
 7701 // Load Pointer Constant One
 7702 
 7703 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7704 %{
 7705   match(Set dst con);
 7706 
 7707   ins_cost(INSN_COST);
 7708   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7709 
 7710   ins_encode(aarch64_enc_mov_p1(dst, con));
 7711 
 7712   ins_pipe(ialu_imm);
 7713 %}
 7714 
 7715 // Load Byte Map Base Constant
 7716 
 7717 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7718 %{
 7719   match(Set dst con);
 7720 
 7721   ins_cost(INSN_COST);
 7722   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7723 
 7724   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7725 
 7726   ins_pipe(ialu_imm);
 7727 %}
 7728 
 7729 // Load Narrow Pointer Constant
 7730 
 7731 instruct loadConN(iRegNNoSp dst, immN con)
 7732 %{
 7733   match(Set dst con);
 7734 
 7735   ins_cost(INSN_COST * 4);
 7736   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7737 
 7738   ins_encode(aarch64_enc_mov_n(dst, con));
 7739 
 7740   ins_pipe(ialu_imm);
 7741 %}
 7742 
 7743 // Load Narrow Null Pointer Constant
 7744 
 7745 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7746 %{
 7747   match(Set dst con);
 7748 
 7749   ins_cost(INSN_COST);
 7750   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7751 
 7752   ins_encode(aarch64_enc_mov_n0(dst, con));
 7753 
 7754   ins_pipe(ialu_imm);
 7755 %}
 7756 
 7757 // Load Narrow Klass Constant
 7758 
 7759 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7760 %{
 7761   match(Set dst con);
 7762 
 7763   ins_cost(INSN_COST);
 7764   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7765 
 7766   ins_encode(aarch64_enc_mov_nk(dst, con));
 7767 
 7768   ins_pipe(ialu_imm);
 7769 %}
 7770 
 7771 // Load Packed Float Constant
 7772 
 7773 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7774   match(Set dst con);
 7775   ins_cost(INSN_COST * 4);
 7776   format %{ "fmovs  $dst, $con"%}
 7777   ins_encode %{
 7778     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7779   %}
 7780 
 7781   ins_pipe(fp_imm_s);
 7782 %}
 7783 
 7784 // Load Float Constant
 7785 
 7786 instruct loadConF(vRegF dst, immF con) %{
 7787   match(Set dst con);
 7788 
 7789   ins_cost(INSN_COST * 4);
 7790 
 7791   format %{
 7792     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7793   %}
 7794 
 7795   ins_encode %{
 7796     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7797   %}
 7798 
 7799   ins_pipe(fp_load_constant_s);
 7800 %}
 7801 
 7802 // Load Packed Double Constant
 7803 
 7804 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7805   match(Set dst con);
 7806   ins_cost(INSN_COST);
 7807   format %{ "fmovd  $dst, $con"%}
 7808   ins_encode %{
 7809     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7810   %}
 7811 
 7812   ins_pipe(fp_imm_d);
 7813 %}
 7814 
 7815 // Load Double Constant
 7816 
 7817 instruct loadConD(vRegD dst, immD con) %{
 7818   match(Set dst con);
 7819 
 7820   ins_cost(INSN_COST * 5);
 7821   format %{
 7822     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7823   %}
 7824 
 7825   ins_encode %{
 7826     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7827   %}
 7828 
 7829   ins_pipe(fp_load_constant_d);
 7830 %}
 7831 
 7832 // Store Instructions
 7833 
 7834 // Store CMS card-mark Immediate
 7835 instruct storeimmCM0(immI0 zero, memory1 mem)
 7836 %{
 7837   match(Set mem (StoreCM mem zero));
 7838 
 7839   ins_cost(INSN_COST);
 7840   format %{ "storestore (elided)\n\t"
 7841             "strb zr, $mem\t# byte" %}
 7842 
 7843   ins_encode(aarch64_enc_strb0(mem));
 7844 
 7845   ins_pipe(istore_mem);
 7846 %}
 7847 
 7848 // Store CMS card-mark Immediate with intervening StoreStore
 7849 // needed when using CMS with no conditional card marking
 7850 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7851 %{
 7852   match(Set mem (StoreCM mem zero));
 7853 
 7854   ins_cost(INSN_COST * 2);
 7855   format %{ "storestore\n\t"
 7856             "dmb ishst"
 7857             "\n\tstrb zr, $mem\t# byte" %}
 7858 
 7859   ins_encode(aarch64_enc_strb0_ordered(mem));
 7860 
 7861   ins_pipe(istore_mem);
 7862 %}
 7863 
 7864 // Store Byte
 7865 instruct storeB(iRegIorL2I src, memory1 mem)
 7866 %{
 7867   match(Set mem (StoreB mem src));
 7868   predicate(!needs_releasing_store(n));
 7869 
 7870   ins_cost(INSN_COST);
 7871   format %{ "strb  $src, $mem\t# byte" %}
 7872 
 7873   ins_encode(aarch64_enc_strb(src, mem));
 7874 
 7875   ins_pipe(istore_reg_mem);
 7876 %}
 7877 
 7878 
 7879 instruct storeimmB0(immI0 zero, memory1 mem)
 7880 %{
 7881   match(Set mem (StoreB mem zero));
 7882   predicate(!needs_releasing_store(n));
 7883 
 7884   ins_cost(INSN_COST);
 7885   format %{ "strb rscractch2, $mem\t# byte" %}
 7886 
 7887   ins_encode(aarch64_enc_strb0(mem));
 7888 
 7889   ins_pipe(istore_mem);
 7890 %}
 7891 
 7892 // Store Char/Short
 7893 instruct storeC(iRegIorL2I src, memory2 mem)
 7894 %{
 7895   match(Set mem (StoreC mem src));
 7896   predicate(!needs_releasing_store(n));
 7897 
 7898   ins_cost(INSN_COST);
 7899   format %{ "strh  $src, $mem\t# short" %}
 7900 
 7901   ins_encode(aarch64_enc_strh(src, mem));
 7902 
 7903   ins_pipe(istore_reg_mem);
 7904 %}
 7905 
 7906 instruct storeimmC0(immI0 zero, memory2 mem)
 7907 %{
 7908   match(Set mem (StoreC mem zero));
 7909   predicate(!needs_releasing_store(n));
 7910 
 7911   ins_cost(INSN_COST);
 7912   format %{ "strh  zr, $mem\t# short" %}
 7913 
 7914   ins_encode(aarch64_enc_strh0(mem));
 7915 
 7916   ins_pipe(istore_mem);
 7917 %}
 7918 
 7919 // Store Integer
 7920 
 7921 instruct storeI(iRegIorL2I src, memory4 mem)
 7922 %{
 7923   match(Set mem(StoreI mem src));
 7924   predicate(!needs_releasing_store(n));
 7925 
 7926   ins_cost(INSN_COST);
 7927   format %{ "strw  $src, $mem\t# int" %}
 7928 
 7929   ins_encode(aarch64_enc_strw(src, mem));
 7930 
 7931   ins_pipe(istore_reg_mem);
 7932 %}
 7933 
 7934 instruct storeimmI0(immI0 zero, memory4 mem)
 7935 %{
 7936   match(Set mem(StoreI mem zero));
 7937   predicate(!needs_releasing_store(n));
 7938 
 7939   ins_cost(INSN_COST);
 7940   format %{ "strw  zr, $mem\t# int" %}
 7941 
 7942   ins_encode(aarch64_enc_strw0(mem));
 7943 
 7944   ins_pipe(istore_mem);
 7945 %}
 7946 
 7947 // Store Long (64 bit signed)
 7948 instruct storeL(iRegL src, memory8 mem)
 7949 %{
 7950   match(Set mem (StoreL mem src));
 7951   predicate(!needs_releasing_store(n));
 7952 
 7953   ins_cost(INSN_COST);
 7954   format %{ "str  $src, $mem\t# int" %}
 7955 
 7956   ins_encode(aarch64_enc_str(src, mem));
 7957 
 7958   ins_pipe(istore_reg_mem);
 7959 %}
 7960 
 7961 // Store Long (64 bit signed)
 7962 instruct storeimmL0(immL0 zero, memory8 mem)
 7963 %{
 7964   match(Set mem (StoreL mem zero));
 7965   predicate(!needs_releasing_store(n));
 7966 
 7967   ins_cost(INSN_COST);
 7968   format %{ "str  zr, $mem\t# int" %}
 7969 
 7970   ins_encode(aarch64_enc_str0(mem));
 7971 
 7972   ins_pipe(istore_mem);
 7973 %}
 7974 
 7975 // Store Pointer
 7976 instruct storeP(iRegP src, memory8 mem)
 7977 %{
 7978   match(Set mem (StoreP mem src));
 7979   predicate(!needs_releasing_store(n));
 7980 
 7981   ins_cost(INSN_COST);
 7982   format %{ "str  $src, $mem\t# ptr" %}
 7983 
 7984   ins_encode(aarch64_enc_str(src, mem));
 7985 
 7986   ins_pipe(istore_reg_mem);
 7987 %}
 7988 
 7989 // Store Pointer
 7990 instruct storeimmP0(immP0 zero, memory8 mem)
 7991 %{
 7992   match(Set mem (StoreP mem zero));
 7993   predicate(!needs_releasing_store(n));
 7994 
 7995   ins_cost(INSN_COST);
 7996   format %{ "str zr, $mem\t# ptr" %}
 7997 
 7998   ins_encode(aarch64_enc_str0(mem));
 7999 
 8000   ins_pipe(istore_mem);
 8001 %}
 8002 
 8003 // Store Compressed Pointer
 8004 instruct storeN(iRegN src, memory4 mem)
 8005 %{
 8006   match(Set mem (StoreN mem src));
 8007   predicate(!needs_releasing_store(n));
 8008 
 8009   ins_cost(INSN_COST);
 8010   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8011 
 8012   ins_encode(aarch64_enc_strw(src, mem));
 8013 
 8014   ins_pipe(istore_reg_mem);
 8015 %}
 8016 
 8017 instruct storeImmN0(immN0 zero, memory4 mem)
 8018 %{
 8019   match(Set mem (StoreN mem zero));
 8020   predicate(!needs_releasing_store(n));
 8021 
 8022   ins_cost(INSN_COST);
 8023   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8024 
 8025   ins_encode(aarch64_enc_strw0(mem));
 8026 
 8027   ins_pipe(istore_mem);
 8028 %}
 8029 
 8030 // Store Float
 8031 instruct storeF(vRegF src, memory4 mem)
 8032 %{
 8033   match(Set mem (StoreF mem src));
 8034   predicate(!needs_releasing_store(n));
 8035 
 8036   ins_cost(INSN_COST);
 8037   format %{ "strs  $src, $mem\t# float" %}
 8038 
 8039   ins_encode( aarch64_enc_strs(src, mem) );
 8040 
 8041   ins_pipe(pipe_class_memory);
 8042 %}
 8043 
 8044 // TODO
 8045 // implement storeImmF0 and storeFImmPacked
 8046 
 8047 // Store Double
 8048 instruct storeD(vRegD src, memory8 mem)
 8049 %{
 8050   match(Set mem (StoreD mem src));
 8051   predicate(!needs_releasing_store(n));
 8052 
 8053   ins_cost(INSN_COST);
 8054   format %{ "strd  $src, $mem\t# double" %}
 8055 
 8056   ins_encode( aarch64_enc_strd(src, mem) );
 8057 
 8058   ins_pipe(pipe_class_memory);
 8059 %}
 8060 
 8061 // Store Compressed Klass Pointer
 8062 instruct storeNKlass(iRegN src, memory4 mem)
 8063 %{
 8064   predicate(!needs_releasing_store(n));
 8065   match(Set mem (StoreNKlass mem src));
 8066 
 8067   ins_cost(INSN_COST);
 8068   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8069 
 8070   ins_encode(aarch64_enc_strw(src, mem));
 8071 
 8072   ins_pipe(istore_reg_mem);
 8073 %}
 8074 
 8075 // TODO
 8076 // implement storeImmD0 and storeDImmPacked
 8077 
 8078 // prefetch instructions
 8079 // Must be safe to execute with invalid address (cannot fault).
 8080 
 8081 instruct prefetchalloc( memory8 mem ) %{
 8082   match(PrefetchAllocation mem);
 8083 
 8084   ins_cost(INSN_COST);
 8085   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8086 
 8087   ins_encode( aarch64_enc_prefetchw(mem) );
 8088 
 8089   ins_pipe(iload_prefetch);
 8090 %}
 8091 
 8092 //  ---------------- volatile loads and stores ----------------
 8093 
 8094 // Load Byte (8 bit signed)
 8095 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8096 %{
 8097   match(Set dst (LoadB mem));
 8098 
 8099   ins_cost(VOLATILE_REF_COST);
 8100   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8101 
 8102   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8103 
 8104   ins_pipe(pipe_serial);
 8105 %}
 8106 
 8107 // Load Byte (8 bit signed) into long
 8108 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8109 %{
 8110   match(Set dst (ConvI2L (LoadB mem)));
 8111 
 8112   ins_cost(VOLATILE_REF_COST);
 8113   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8114 
 8115   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8116 
 8117   ins_pipe(pipe_serial);
 8118 %}
 8119 
 8120 // Load Byte (8 bit unsigned)
 8121 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8122 %{
 8123   match(Set dst (LoadUB mem));
 8124 
 8125   ins_cost(VOLATILE_REF_COST);
 8126   format %{ "ldarb  $dst, $mem\t# byte" %}
 8127 
 8128   ins_encode(aarch64_enc_ldarb(dst, mem));
 8129 
 8130   ins_pipe(pipe_serial);
 8131 %}
 8132 
 8133 // Load Byte (8 bit unsigned) into long
 8134 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8135 %{
 8136   match(Set dst (ConvI2L (LoadUB mem)));
 8137 
 8138   ins_cost(VOLATILE_REF_COST);
 8139   format %{ "ldarb  $dst, $mem\t# byte" %}
 8140 
 8141   ins_encode(aarch64_enc_ldarb(dst, mem));
 8142 
 8143   ins_pipe(pipe_serial);
 8144 %}
 8145 
 8146 // Load Short (16 bit signed)
 8147 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8148 %{
 8149   match(Set dst (LoadS mem));
 8150 
 8151   ins_cost(VOLATILE_REF_COST);
 8152   format %{ "ldarshw  $dst, $mem\t# short" %}
 8153 
 8154   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8155 
 8156   ins_pipe(pipe_serial);
 8157 %}
 8158 
 8159 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8160 %{
 8161   match(Set dst (LoadUS mem));
 8162 
 8163   ins_cost(VOLATILE_REF_COST);
 8164   format %{ "ldarhw  $dst, $mem\t# short" %}
 8165 
 8166   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8167 
 8168   ins_pipe(pipe_serial);
 8169 %}
 8170 
 8171 // Load Short/Char (16 bit unsigned) into long
 8172 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8173 %{
 8174   match(Set dst (ConvI2L (LoadUS mem)));
 8175 
 8176   ins_cost(VOLATILE_REF_COST);
 8177   format %{ "ldarh  $dst, $mem\t# short" %}
 8178 
 8179   ins_encode(aarch64_enc_ldarh(dst, mem));
 8180 
 8181   ins_pipe(pipe_serial);
 8182 %}
 8183 
 8184 // Load Short/Char (16 bit signed) into long
 8185 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8186 %{
 8187   match(Set dst (ConvI2L (LoadS mem)));
 8188 
 8189   ins_cost(VOLATILE_REF_COST);
 8190   format %{ "ldarh  $dst, $mem\t# short" %}
 8191 
 8192   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8193 
 8194   ins_pipe(pipe_serial);
 8195 %}
 8196 
 8197 // Load Integer (32 bit signed)
 8198 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8199 %{
 8200   match(Set dst (LoadI mem));
 8201 
 8202   ins_cost(VOLATILE_REF_COST);
 8203   format %{ "ldarw  $dst, $mem\t# int" %}
 8204 
 8205   ins_encode(aarch64_enc_ldarw(dst, mem));
 8206 
 8207   ins_pipe(pipe_serial);
 8208 %}
 8209 
 8210 // Load Integer (32 bit unsigned) into long
 8211 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8212 %{
 8213   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8214 
 8215   ins_cost(VOLATILE_REF_COST);
 8216   format %{ "ldarw  $dst, $mem\t# int" %}
 8217 
 8218   ins_encode(aarch64_enc_ldarw(dst, mem));
 8219 
 8220   ins_pipe(pipe_serial);
 8221 %}
 8222 
 8223 // Load Long (64 bit signed)
 8224 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8225 %{
 8226   match(Set dst (LoadL mem));
 8227 
 8228   ins_cost(VOLATILE_REF_COST);
 8229   format %{ "ldar  $dst, $mem\t# int" %}
 8230 
 8231   ins_encode(aarch64_enc_ldar(dst, mem));
 8232 
 8233   ins_pipe(pipe_serial);
 8234 %}
 8235 
 8236 // Load Pointer
 8237 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8238 %{
 8239   match(Set dst (LoadP mem));
 8240   predicate(n->as_Load()->barrier_data() == 0);
 8241 
 8242   ins_cost(VOLATILE_REF_COST);
 8243   format %{ "ldar  $dst, $mem\t# ptr" %}
 8244 
 8245   ins_encode(aarch64_enc_ldar(dst, mem));
 8246 
 8247   ins_pipe(pipe_serial);
 8248 %}
 8249 
 8250 // Load Compressed Pointer
 8251 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8252 %{
 8253   match(Set dst (LoadN mem));
 8254 
 8255   ins_cost(VOLATILE_REF_COST);
 8256   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8257 
 8258   ins_encode(aarch64_enc_ldarw(dst, mem));
 8259 
 8260   ins_pipe(pipe_serial);
 8261 %}
 8262 
 8263 // Load Float
 8264 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8265 %{
 8266   match(Set dst (LoadF mem));
 8267 
 8268   ins_cost(VOLATILE_REF_COST);
 8269   format %{ "ldars  $dst, $mem\t# float" %}
 8270 
 8271   ins_encode( aarch64_enc_fldars(dst, mem) );
 8272 
 8273   ins_pipe(pipe_serial);
 8274 %}
 8275 
 8276 // Load Double
 8277 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8278 %{
 8279   match(Set dst (LoadD mem));
 8280 
 8281   ins_cost(VOLATILE_REF_COST);
 8282   format %{ "ldard  $dst, $mem\t# double" %}
 8283 
 8284   ins_encode( aarch64_enc_fldard(dst, mem) );
 8285 
 8286   ins_pipe(pipe_serial);
 8287 %}
 8288 
 8289 // Store Byte
 8290 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8291 %{
 8292   match(Set mem (StoreB mem src));
 8293 
 8294   ins_cost(VOLATILE_REF_COST);
 8295   format %{ "stlrb  $src, $mem\t# byte" %}
 8296 
 8297   ins_encode(aarch64_enc_stlrb(src, mem));
 8298 
 8299   ins_pipe(pipe_class_memory);
 8300 %}
 8301 
 8302 // Store Char/Short
 8303 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8304 %{
 8305   match(Set mem (StoreC mem src));
 8306 
 8307   ins_cost(VOLATILE_REF_COST);
 8308   format %{ "stlrh  $src, $mem\t# short" %}
 8309 
 8310   ins_encode(aarch64_enc_stlrh(src, mem));
 8311 
 8312   ins_pipe(pipe_class_memory);
 8313 %}
 8314 
 8315 // Store Integer
 8316 
 8317 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8318 %{
 8319   match(Set mem(StoreI mem src));
 8320 
 8321   ins_cost(VOLATILE_REF_COST);
 8322   format %{ "stlrw  $src, $mem\t# int" %}
 8323 
 8324   ins_encode(aarch64_enc_stlrw(src, mem));
 8325 
 8326   ins_pipe(pipe_class_memory);
 8327 %}
 8328 
 8329 // Store Long (64 bit signed)
 8330 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8331 %{
 8332   match(Set mem (StoreL mem src));
 8333 
 8334   ins_cost(VOLATILE_REF_COST);
 8335   format %{ "stlr  $src, $mem\t# int" %}
 8336 
 8337   ins_encode(aarch64_enc_stlr(src, mem));
 8338 
 8339   ins_pipe(pipe_class_memory);
 8340 %}
 8341 
 8342 // Store Pointer
 8343 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8344 %{
 8345   match(Set mem (StoreP mem src));
 8346 
 8347   ins_cost(VOLATILE_REF_COST);
 8348   format %{ "stlr  $src, $mem\t# ptr" %}
 8349 
 8350   ins_encode(aarch64_enc_stlr(src, mem));
 8351 
 8352   ins_pipe(pipe_class_memory);
 8353 %}
 8354 
 8355 // Store Compressed Pointer
 8356 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8357 %{
 8358   match(Set mem (StoreN mem src));
 8359 
 8360   ins_cost(VOLATILE_REF_COST);
 8361   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8362 
 8363   ins_encode(aarch64_enc_stlrw(src, mem));
 8364 
 8365   ins_pipe(pipe_class_memory);
 8366 %}
 8367 
 8368 // Store Float
 8369 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8370 %{
 8371   match(Set mem (StoreF mem src));
 8372 
 8373   ins_cost(VOLATILE_REF_COST);
 8374   format %{ "stlrs  $src, $mem\t# float" %}
 8375 
 8376   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8377 
 8378   ins_pipe(pipe_class_memory);
 8379 %}
 8380 
 8381 // TODO
 8382 // implement storeImmF0 and storeFImmPacked
 8383 
 8384 // Store Double
 8385 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8386 %{
 8387   match(Set mem (StoreD mem src));
 8388 
 8389   ins_cost(VOLATILE_REF_COST);
 8390   format %{ "stlrd  $src, $mem\t# double" %}
 8391 
 8392   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8393 
 8394   ins_pipe(pipe_class_memory);
 8395 %}
 8396 
 8397 //  ---------------- end of volatile loads and stores ----------------
 8398 
 8399 instruct cacheWB(indirect addr)
 8400 %{
 8401   predicate(VM_Version::supports_data_cache_line_flush());
 8402   match(CacheWB addr);
 8403 
 8404   ins_cost(100);
 8405   format %{"cache wb $addr" %}
 8406   ins_encode %{
 8407     assert($addr->index_position() < 0, "should be");
 8408     assert($addr$$disp == 0, "should be");
 8409     __ cache_wb(Address($addr$$base$$Register, 0));
 8410   %}
 8411   ins_pipe(pipe_slow); // XXX
 8412 %}
 8413 
 8414 instruct cacheWBPreSync()
 8415 %{
 8416   predicate(VM_Version::supports_data_cache_line_flush());
 8417   match(CacheWBPreSync);
 8418 
 8419   ins_cost(100);
 8420   format %{"cache wb presync" %}
 8421   ins_encode %{
 8422     __ cache_wbsync(true);
 8423   %}
 8424   ins_pipe(pipe_slow); // XXX
 8425 %}
 8426 
 8427 instruct cacheWBPostSync()
 8428 %{
 8429   predicate(VM_Version::supports_data_cache_line_flush());
 8430   match(CacheWBPostSync);
 8431 
 8432   ins_cost(100);
 8433   format %{"cache wb postsync" %}
 8434   ins_encode %{
 8435     __ cache_wbsync(false);
 8436   %}
 8437   ins_pipe(pipe_slow); // XXX
 8438 %}
 8439 
 8440 // ============================================================================
 8441 // BSWAP Instructions
 8442 
 8443 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8444   match(Set dst (ReverseBytesI src));
 8445 
 8446   ins_cost(INSN_COST);
 8447   format %{ "revw  $dst, $src" %}
 8448 
 8449   ins_encode %{
 8450     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8451   %}
 8452 
 8453   ins_pipe(ialu_reg);
 8454 %}
 8455 
 8456 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8457   match(Set dst (ReverseBytesL src));
 8458 
 8459   ins_cost(INSN_COST);
 8460   format %{ "rev  $dst, $src" %}
 8461 
 8462   ins_encode %{
 8463     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8464   %}
 8465 
 8466   ins_pipe(ialu_reg);
 8467 %}
 8468 
 8469 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8470   match(Set dst (ReverseBytesUS src));
 8471 
 8472   ins_cost(INSN_COST);
 8473   format %{ "rev16w  $dst, $src" %}
 8474 
 8475   ins_encode %{
 8476     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8477   %}
 8478 
 8479   ins_pipe(ialu_reg);
 8480 %}
 8481 
 8482 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8483   match(Set dst (ReverseBytesS src));
 8484 
 8485   ins_cost(INSN_COST);
 8486   format %{ "rev16w  $dst, $src\n\t"
 8487             "sbfmw $dst, $dst, #0, #15" %}
 8488 
 8489   ins_encode %{
 8490     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8491     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8492   %}
 8493 
 8494   ins_pipe(ialu_reg);
 8495 %}
 8496 
 8497 // ============================================================================
 8498 // Zero Count Instructions
 8499 
 8500 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8501   match(Set dst (CountLeadingZerosI src));
 8502 
 8503   ins_cost(INSN_COST);
 8504   format %{ "clzw  $dst, $src" %}
 8505   ins_encode %{
 8506     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8507   %}
 8508 
 8509   ins_pipe(ialu_reg);
 8510 %}
 8511 
 8512 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8513   match(Set dst (CountLeadingZerosL src));
 8514 
 8515   ins_cost(INSN_COST);
 8516   format %{ "clz   $dst, $src" %}
 8517   ins_encode %{
 8518     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8519   %}
 8520 
 8521   ins_pipe(ialu_reg);
 8522 %}
 8523 
 8524 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8525   match(Set dst (CountTrailingZerosI src));
 8526 
 8527   ins_cost(INSN_COST * 2);
 8528   format %{ "rbitw  $dst, $src\n\t"
 8529             "clzw   $dst, $dst" %}
 8530   ins_encode %{
 8531     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8532     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8533   %}
 8534 
 8535   ins_pipe(ialu_reg);
 8536 %}
 8537 
 8538 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8539   match(Set dst (CountTrailingZerosL src));
 8540 
 8541   ins_cost(INSN_COST * 2);
 8542   format %{ "rbit   $dst, $src\n\t"
 8543             "clz    $dst, $dst" %}
 8544   ins_encode %{
 8545     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8546     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8547   %}
 8548 
 8549   ins_pipe(ialu_reg);
 8550 %}
 8551 
 8552 //---------- Population Count Instructions -------------------------------------
 8553 //
 8554 
 8555 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8556   predicate(UsePopCountInstruction);
 8557   match(Set dst (PopCountI src));
 8558   effect(TEMP tmp);
 8559   ins_cost(INSN_COST * 13);
 8560 
 8561   format %{ "movw   $src, $src\n\t"
 8562             "mov    $tmp, $src\t# vector (1D)\n\t"
 8563             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8564             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8565             "mov    $dst, $tmp\t# vector (1D)" %}
 8566   ins_encode %{
 8567     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8568     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8569     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8570     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8571     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8572   %}
 8573 
 8574   ins_pipe(pipe_class_default);
 8575 %}
 8576 
 8577 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8578   predicate(UsePopCountInstruction);
 8579   match(Set dst (PopCountI (LoadI mem)));
 8580   effect(TEMP tmp);
 8581   ins_cost(INSN_COST * 13);
 8582 
 8583   format %{ "ldrs   $tmp, $mem\n\t"
 8584             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8585             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8586             "mov    $dst, $tmp\t# vector (1D)" %}
 8587   ins_encode %{
 8588     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8589     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8590               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8591     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8592     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8593     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8594   %}
 8595 
 8596   ins_pipe(pipe_class_default);
 8597 %}
 8598 
 8599 // Note: Long.bitCount(long) returns an int.
 8600 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8601   predicate(UsePopCountInstruction);
 8602   match(Set dst (PopCountL src));
 8603   effect(TEMP tmp);
 8604   ins_cost(INSN_COST * 13);
 8605 
 8606   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8607             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8608             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8609             "mov    $dst, $tmp\t# vector (1D)" %}
 8610   ins_encode %{
 8611     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8612     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8613     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8614     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8615   %}
 8616 
 8617   ins_pipe(pipe_class_default);
 8618 %}
 8619 
 8620 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8621   predicate(UsePopCountInstruction);
 8622   match(Set dst (PopCountL (LoadL mem)));
 8623   effect(TEMP tmp);
 8624   ins_cost(INSN_COST * 13);
 8625 
 8626   format %{ "ldrd   $tmp, $mem\n\t"
 8627             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8628             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8629             "mov    $dst, $tmp\t# vector (1D)" %}
 8630   ins_encode %{
 8631     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8632     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8633               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8634     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8635     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8636     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8637   %}
 8638 
 8639   ins_pipe(pipe_class_default);
 8640 %}
 8641 
 8642 // ============================================================================
 8643 // MemBar Instruction
 8644 
 8645 instruct load_fence() %{
 8646   match(LoadFence);
 8647   ins_cost(VOLATILE_REF_COST);
 8648 
 8649   format %{ "load_fence" %}
 8650 
 8651   ins_encode %{
 8652     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8653   %}
 8654   ins_pipe(pipe_serial);
 8655 %}
 8656 
 8657 instruct unnecessary_membar_acquire() %{
 8658   predicate(unnecessary_acquire(n));
 8659   match(MemBarAcquire);
 8660   ins_cost(0);
 8661 
 8662   format %{ "membar_acquire (elided)" %}
 8663 
 8664   ins_encode %{
 8665     __ block_comment("membar_acquire (elided)");
 8666   %}
 8667 
 8668   ins_pipe(pipe_class_empty);
 8669 %}
 8670 
 8671 instruct membar_acquire() %{
 8672   match(MemBarAcquire);
 8673   ins_cost(VOLATILE_REF_COST);
 8674 
 8675   format %{ "membar_acquire\n\t"
 8676             "dmb ish" %}
 8677 
 8678   ins_encode %{
 8679     __ block_comment("membar_acquire");
 8680     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8681   %}
 8682 
 8683   ins_pipe(pipe_serial);
 8684 %}
 8685 
 8686 
 8687 instruct membar_acquire_lock() %{
 8688   match(MemBarAcquireLock);
 8689   ins_cost(VOLATILE_REF_COST);
 8690 
 8691   format %{ "membar_acquire_lock (elided)" %}
 8692 
 8693   ins_encode %{
 8694     __ block_comment("membar_acquire_lock (elided)");
 8695   %}
 8696 
 8697   ins_pipe(pipe_serial);
 8698 %}
 8699 
 8700 instruct store_fence() %{
 8701   match(StoreFence);
 8702   ins_cost(VOLATILE_REF_COST);
 8703 
 8704   format %{ "store_fence" %}
 8705 
 8706   ins_encode %{
 8707     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8708   %}
 8709   ins_pipe(pipe_serial);
 8710 %}
 8711 
 8712 instruct unnecessary_membar_release() %{
 8713   predicate(unnecessary_release(n));
 8714   match(MemBarRelease);
 8715   ins_cost(0);
 8716 
 8717   format %{ "membar_release (elided)" %}
 8718 
 8719   ins_encode %{
 8720     __ block_comment("membar_release (elided)");
 8721   %}
 8722   ins_pipe(pipe_serial);
 8723 %}
 8724 
 8725 instruct membar_release() %{
 8726   match(MemBarRelease);
 8727   ins_cost(VOLATILE_REF_COST);
 8728 
 8729   format %{ "membar_release\n\t"
 8730             "dmb ish" %}
 8731 
 8732   ins_encode %{
 8733     __ block_comment("membar_release");
 8734     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8735   %}
 8736   ins_pipe(pipe_serial);
 8737 %}
 8738 
 8739 instruct membar_storestore() %{
 8740   match(MemBarStoreStore);
 8741   match(StoreStoreFence);
 8742   ins_cost(VOLATILE_REF_COST);
 8743 
 8744   format %{ "MEMBAR-store-store" %}
 8745 
 8746   ins_encode %{
 8747     __ membar(Assembler::StoreStore);
 8748   %}
 8749   ins_pipe(pipe_serial);
 8750 %}
 8751 
 8752 instruct membar_release_lock() %{
 8753   match(MemBarReleaseLock);
 8754   ins_cost(VOLATILE_REF_COST);
 8755 
 8756   format %{ "membar_release_lock (elided)" %}
 8757 
 8758   ins_encode %{
 8759     __ block_comment("membar_release_lock (elided)");
 8760   %}
 8761 
 8762   ins_pipe(pipe_serial);
 8763 %}
 8764 
 8765 instruct unnecessary_membar_volatile() %{
 8766   predicate(unnecessary_volatile(n));
 8767   match(MemBarVolatile);
 8768   ins_cost(0);
 8769 
 8770   format %{ "membar_volatile (elided)" %}
 8771 
 8772   ins_encode %{
 8773     __ block_comment("membar_volatile (elided)");
 8774   %}
 8775 
 8776   ins_pipe(pipe_serial);
 8777 %}
 8778 
 8779 instruct membar_volatile() %{
 8780   match(MemBarVolatile);
 8781   ins_cost(VOLATILE_REF_COST*100);
 8782 
 8783   format %{ "membar_volatile\n\t"
 8784              "dmb ish"%}
 8785 
 8786   ins_encode %{
 8787     __ block_comment("membar_volatile");
 8788     __ membar(Assembler::StoreLoad);
 8789   %}
 8790 
 8791   ins_pipe(pipe_serial);
 8792 %}
 8793 
 8794 // ============================================================================
 8795 // Cast/Convert Instructions
 8796 
 8797 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8798   match(Set dst (CastX2P src));
 8799 
 8800   ins_cost(INSN_COST);
 8801   format %{ "mov $dst, $src\t# long -> ptr" %}
 8802 
 8803   ins_encode %{
 8804     if ($dst$$reg != $src$$reg) {
 8805       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8806     }
 8807   %}
 8808 
 8809   ins_pipe(ialu_reg);
 8810 %}
 8811 
 8812 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8813   match(Set dst (CastP2X src));
 8814 
 8815   ins_cost(INSN_COST);
 8816   format %{ "mov $dst, $src\t# ptr -> long" %}
 8817 
 8818   ins_encode %{
 8819     if ($dst$$reg != $src$$reg) {
 8820       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8821     }
 8822   %}
 8823 
 8824   ins_pipe(ialu_reg);
 8825 %}
 8826 
 8827 // Convert oop into int for vectors alignment masking
 8828 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8829   match(Set dst (ConvL2I (CastP2X src)));
 8830 
 8831   ins_cost(INSN_COST);
 8832   format %{ "movw $dst, $src\t# ptr -> int" %}
 8833   ins_encode %{
 8834     __ movw($dst$$Register, $src$$Register);
 8835   %}
 8836 
 8837   ins_pipe(ialu_reg);
 8838 %}
 8839 
 8840 // Convert compressed oop into int for vectors alignment masking
 8841 // in case of 32bit oops (heap < 4Gb).
 8842 instruct convN2I(iRegINoSp dst, iRegN src)
 8843 %{
 8844   predicate(CompressedOops::shift() == 0);
 8845   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8846 
 8847   ins_cost(INSN_COST);
 8848   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8849   ins_encode %{
 8850     __ movw($dst$$Register, $src$$Register);
 8851   %}
 8852 
 8853   ins_pipe(ialu_reg);
 8854 %}
 8855 
 8856 
 8857 // Convert oop pointer into compressed form
 8858 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8859   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8860   match(Set dst (EncodeP src));
 8861   effect(KILL cr);
 8862   ins_cost(INSN_COST * 3);
 8863   format %{ "encode_heap_oop $dst, $src" %}
 8864   ins_encode %{
 8865     Register s = $src$$Register;
 8866     Register d = $dst$$Register;
 8867     __ encode_heap_oop(d, s);
 8868   %}
 8869   ins_pipe(ialu_reg);
 8870 %}
 8871 
 8872 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8873   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8874   match(Set dst (EncodeP src));
 8875   ins_cost(INSN_COST * 3);
 8876   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8877   ins_encode %{
 8878     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8879   %}
 8880   ins_pipe(ialu_reg);
 8881 %}
 8882 
 8883 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8884   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8885             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8886   match(Set dst (DecodeN src));
 8887   ins_cost(INSN_COST * 3);
 8888   format %{ "decode_heap_oop $dst, $src" %}
 8889   ins_encode %{
 8890     Register s = $src$$Register;
 8891     Register d = $dst$$Register;
 8892     __ decode_heap_oop(d, s);
 8893   %}
 8894   ins_pipe(ialu_reg);
 8895 %}
 8896 
 8897 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8898   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8899             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8900   match(Set dst (DecodeN src));
 8901   ins_cost(INSN_COST * 3);
 8902   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8903   ins_encode %{
 8904     Register s = $src$$Register;
 8905     Register d = $dst$$Register;
 8906     __ decode_heap_oop_not_null(d, s);
 8907   %}
 8908   ins_pipe(ialu_reg);
 8909 %}
 8910 
 8911 // n.b. AArch64 implementations of encode_klass_not_null and
 8912 // decode_klass_not_null do not modify the flags register so, unlike
 8913 // Intel, we don't kill CR as a side effect here
 8914 
 8915 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8916   match(Set dst (EncodePKlass src));
 8917 
 8918   ins_cost(INSN_COST * 3);
 8919   format %{ "encode_klass_not_null $dst,$src" %}
 8920 
 8921   ins_encode %{
 8922     Register src_reg = as_Register($src$$reg);
 8923     Register dst_reg = as_Register($dst$$reg);
 8924     __ encode_klass_not_null(dst_reg, src_reg);
 8925   %}
 8926 
 8927    ins_pipe(ialu_reg);
 8928 %}
 8929 
 8930 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8931   match(Set dst (DecodeNKlass src));
 8932 
 8933   ins_cost(INSN_COST * 3);
 8934   format %{ "decode_klass_not_null $dst,$src" %}
 8935 
 8936   ins_encode %{
 8937     Register src_reg = as_Register($src$$reg);
 8938     Register dst_reg = as_Register($dst$$reg);
 8939     if (dst_reg != src_reg) {
 8940       __ decode_klass_not_null(dst_reg, src_reg);
 8941     } else {
 8942       __ decode_klass_not_null(dst_reg);
 8943     }
 8944   %}
 8945 
 8946    ins_pipe(ialu_reg);
 8947 %}
 8948 
 8949 instruct checkCastPP(iRegPNoSp dst)
 8950 %{
 8951   match(Set dst (CheckCastPP dst));
 8952 
 8953   size(0);
 8954   format %{ "# checkcastPP of $dst" %}
 8955   ins_encode(/* empty encoding */);
 8956   ins_pipe(pipe_class_empty);
 8957 %}
 8958 
 8959 instruct castPP(iRegPNoSp dst)
 8960 %{
 8961   match(Set dst (CastPP dst));
 8962 
 8963   size(0);
 8964   format %{ "# castPP of $dst" %}
 8965   ins_encode(/* empty encoding */);
 8966   ins_pipe(pipe_class_empty);
 8967 %}
 8968 
 8969 instruct castII(iRegI dst)
 8970 %{
 8971   match(Set dst (CastII dst));
 8972 
 8973   size(0);
 8974   format %{ "# castII of $dst" %}
 8975   ins_encode(/* empty encoding */);
 8976   ins_cost(0);
 8977   ins_pipe(pipe_class_empty);
 8978 %}
 8979 
 8980 instruct castLL(iRegL dst)
 8981 %{
 8982   match(Set dst (CastLL dst));
 8983 
 8984   size(0);
 8985   format %{ "# castLL of $dst" %}
 8986   ins_encode(/* empty encoding */);
 8987   ins_cost(0);
 8988   ins_pipe(pipe_class_empty);
 8989 %}
 8990 
 8991 instruct castFF(vRegF dst)
 8992 %{
 8993   match(Set dst (CastFF dst));
 8994 
 8995   size(0);
 8996   format %{ "# castFF of $dst" %}
 8997   ins_encode(/* empty encoding */);
 8998   ins_cost(0);
 8999   ins_pipe(pipe_class_empty);
 9000 %}
 9001 
 9002 instruct castDD(vRegD dst)
 9003 %{
 9004   match(Set dst (CastDD dst));
 9005 
 9006   size(0);
 9007   format %{ "# castDD of $dst" %}
 9008   ins_encode(/* empty encoding */);
 9009   ins_cost(0);
 9010   ins_pipe(pipe_class_empty);
 9011 %}
 9012 
 9013 instruct castVVD(vecD dst)
 9014 %{
 9015   match(Set dst (CastVV dst));
 9016 
 9017   size(0);
 9018   format %{ "# castVV of $dst" %}
 9019   ins_encode(/* empty encoding */);
 9020   ins_cost(0);
 9021   ins_pipe(pipe_class_empty);
 9022 %}
 9023 
 9024 instruct castVVX(vecX dst)
 9025 %{
 9026   match(Set dst (CastVV dst));
 9027 
 9028   size(0);
 9029   format %{ "# castVV of $dst" %}
 9030   ins_encode(/* empty encoding */);
 9031   ins_cost(0);
 9032   ins_pipe(pipe_class_empty);
 9033 %}
 9034 
 9035 instruct castVV(vReg dst)
 9036 %{
 9037   match(Set dst (CastVV dst));
 9038 
 9039   size(0);
 9040   format %{ "# castVV of $dst" %}
 9041   ins_encode(/* empty encoding */);
 9042   ins_cost(0);
 9043   ins_pipe(pipe_class_empty);
 9044 %}
 9045 
 9046 instruct castVVMask(pRegGov dst)
 9047 %{
 9048   match(Set dst (CastVV dst));
 9049 
 9050   size(0);
 9051   format %{ "# castVV of $dst" %}
 9052   ins_encode(/* empty encoding */);
 9053   ins_cost(0);
 9054   ins_pipe(pipe_class_empty);
 9055 %}
 9056 
 9057 // ============================================================================
 9058 // Atomic operation instructions
 9059 //
 9060 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9061 // Store{PIL}Conditional instructions using a normal load for the
 9062 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9063 //
 9064 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9065 // pair to lock object allocations from Eden space when not using
 9066 // TLABs.
 9067 //
 9068 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9069 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9070 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9071 // only for 64-bit.
 9072 //
 9073 // We implement LoadPLocked and StorePLocked instructions using,
 9074 // respectively the AArch64 hw load-exclusive and store-conditional
 9075 // instructions. Whereas we must implement each of
 9076 // Store{IL}Conditional using a CAS which employs a pair of
 9077 // instructions comprising a load-exclusive followed by a
 9078 // store-conditional.
 9079 
 9080 
 9081 // Locked-load (linked load) of the current heap-top
 9082 // used when updating the eden heap top
 9083 // implemented using ldaxr on AArch64
 9084 
 9085 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9086 %{
 9087   match(Set dst (LoadPLocked mem));
 9088 
 9089   ins_cost(VOLATILE_REF_COST);
 9090 
 9091   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9092 
 9093   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9094 
 9095   ins_pipe(pipe_serial);
 9096 %}
 9097 
 9098 // Conditional-store of the updated heap-top.
 9099 // Used during allocation of the shared heap.
 9100 // Sets flag (EQ) on success.
 9101 // implemented using stlxr on AArch64.
 9102 
 9103 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9104 %{
 9105   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9106 
 9107   ins_cost(VOLATILE_REF_COST);
 9108 
 9109  // TODO
 9110  // do we need to do a store-conditional release or can we just use a
 9111  // plain store-conditional?
 9112 
 9113   format %{
 9114     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9115     "cmpw rscratch1, zr\t# EQ on successful write"
 9116   %}
 9117 
 9118   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9119 
 9120   ins_pipe(pipe_serial);
 9121 %}
 9122 
 9123 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9124 %{
 9125   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9126 
 9127   ins_cost(VOLATILE_REF_COST);
 9128 
 9129   format %{
 9130     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9131     "cmpw rscratch1, zr\t# EQ on successful write"
 9132   %}
 9133 
 9134   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9135 
 9136   ins_pipe(pipe_slow);
 9137 %}
 9138 
 9139 // storeIConditional also has acquire semantics, for no better reason
 9140 // than matching storeLConditional.  At the time of writing this
 9141 // comment storeIConditional was not used anywhere by AArch64.
 9142 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9143 %{
 9144   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9145 
 9146   ins_cost(VOLATILE_REF_COST);
 9147 
 9148   format %{
 9149     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9150     "cmpw rscratch1, zr\t# EQ on successful write"
 9151   %}
 9152 
 9153   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9154 
 9155   ins_pipe(pipe_slow);
 9156 %}
 9157 
 9158 // standard CompareAndSwapX when we are using barriers
 9159 // these have higher priority than the rules selected by a predicate
 9160 
 9161 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9162 // can't match them
 9163 
 9164 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9165 
 9166   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9167   ins_cost(2 * VOLATILE_REF_COST);
 9168 
 9169   effect(KILL cr);
 9170 
 9171   format %{
 9172     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9173     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9174   %}
 9175 
 9176   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9177             aarch64_enc_cset_eq(res));
 9178 
 9179   ins_pipe(pipe_slow);
 9180 %}
 9181 
 9182 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9183 
 9184   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9185   ins_cost(2 * VOLATILE_REF_COST);
 9186 
 9187   effect(KILL cr);
 9188 
 9189   format %{
 9190     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9191     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9192   %}
 9193 
 9194   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9195             aarch64_enc_cset_eq(res));
 9196 
 9197   ins_pipe(pipe_slow);
 9198 %}
 9199 
 9200 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9201 
 9202   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9203   ins_cost(2 * VOLATILE_REF_COST);
 9204 
 9205   effect(KILL cr);
 9206 
 9207  format %{
 9208     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9209     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9210  %}
 9211 
 9212  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9213             aarch64_enc_cset_eq(res));
 9214 
 9215   ins_pipe(pipe_slow);
 9216 %}
 9217 
 9218 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9219 
 9220   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9221   ins_cost(2 * VOLATILE_REF_COST);
 9222 
 9223   effect(KILL cr);
 9224 
 9225  format %{
 9226     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9227     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9228  %}
 9229 
 9230  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9231             aarch64_enc_cset_eq(res));
 9232 
 9233   ins_pipe(pipe_slow);
 9234 %}
 9235 
 9236 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9237 
 9238   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9239   predicate(n->as_LoadStore()->barrier_data() == 0);
 9240   ins_cost(2 * VOLATILE_REF_COST);
 9241 
 9242   effect(KILL cr);
 9243 
 9244  format %{
 9245     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9246     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9247  %}
 9248 
 9249  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9250             aarch64_enc_cset_eq(res));
 9251 
 9252   ins_pipe(pipe_slow);
 9253 %}
 9254 
 9255 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9256 
 9257   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9258   ins_cost(2 * VOLATILE_REF_COST);
 9259 
 9260   effect(KILL cr);
 9261 
 9262  format %{
 9263     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9264     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9265  %}
 9266 
 9267  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9268             aarch64_enc_cset_eq(res));
 9269 
 9270   ins_pipe(pipe_slow);
 9271 %}
 9272 
 9273 // alternative CompareAndSwapX when we are eliding barriers
 9274 
 9275 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9276 
 9277   predicate(needs_acquiring_load_exclusive(n));
 9278   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9279   ins_cost(VOLATILE_REF_COST);
 9280 
 9281   effect(KILL cr);
 9282 
 9283   format %{
 9284     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9285     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9286   %}
 9287 
 9288   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9289             aarch64_enc_cset_eq(res));
 9290 
 9291   ins_pipe(pipe_slow);
 9292 %}
 9293 
 9294 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9295 
 9296   predicate(needs_acquiring_load_exclusive(n));
 9297   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9298   ins_cost(VOLATILE_REF_COST);
 9299 
 9300   effect(KILL cr);
 9301 
 9302   format %{
 9303     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9304     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9305   %}
 9306 
 9307   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9308             aarch64_enc_cset_eq(res));
 9309 
 9310   ins_pipe(pipe_slow);
 9311 %}
 9312 
 9313 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9314 
 9315   predicate(needs_acquiring_load_exclusive(n));
 9316   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9317   ins_cost(VOLATILE_REF_COST);
 9318 
 9319   effect(KILL cr);
 9320 
 9321  format %{
 9322     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9323     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9324  %}
 9325 
 9326  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9327             aarch64_enc_cset_eq(res));
 9328 
 9329   ins_pipe(pipe_slow);
 9330 %}
 9331 
 9332 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9333 
 9334   predicate(needs_acquiring_load_exclusive(n));
 9335   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9336   ins_cost(VOLATILE_REF_COST);
 9337 
 9338   effect(KILL cr);
 9339 
 9340  format %{
 9341     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9342     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9343  %}
 9344 
 9345  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9346             aarch64_enc_cset_eq(res));
 9347 
 9348   ins_pipe(pipe_slow);
 9349 %}
 9350 
 9351 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9352 
 9353   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9354   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9355   ins_cost(VOLATILE_REF_COST);
 9356 
 9357   effect(KILL cr);
 9358 
 9359  format %{
 9360     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9361     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9362  %}
 9363 
 9364  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9365             aarch64_enc_cset_eq(res));
 9366 
 9367   ins_pipe(pipe_slow);
 9368 %}
 9369 
 9370 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9371 
 9372   predicate(needs_acquiring_load_exclusive(n));
 9373   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9374   ins_cost(VOLATILE_REF_COST);
 9375 
 9376   effect(KILL cr);
 9377 
 9378  format %{
 9379     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9380     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9381  %}
 9382 
 9383  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9384             aarch64_enc_cset_eq(res));
 9385 
 9386   ins_pipe(pipe_slow);
 9387 %}
 9388 
 9389 
 9390 // ---------------------------------------------------------------------
 9391 
 9392 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9393 
 9394 // Sundry CAS operations.  Note that release is always true,
 9395 // regardless of the memory ordering of the CAS.  This is because we
 9396 // need the volatile case to be sequentially consistent but there is
 9397 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9398 // can't check the type of memory ordering here, so we always emit a
 9399 // STLXR.
 9400 
 9401 // This section is generated from aarch64_ad_cas.m4
 9402 
 9403 
 9404 
 9405 // This pattern is generated automatically from cas.m4.
 9406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9407 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9408 
 9409   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9410   ins_cost(2 * VOLATILE_REF_COST);
 9411   effect(TEMP_DEF res, KILL cr);
 9412   format %{
 9413     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9414   %}
 9415   ins_encode %{
 9416     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9417                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9418                /*weak*/ false, $res$$Register);
 9419     __ sxtbw($res$$Register, $res$$Register);
 9420   %}
 9421   ins_pipe(pipe_slow);
 9422 %}
 9423 
 9424 // This pattern is generated automatically from cas.m4.
 9425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9426 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9427 
 9428   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9429   ins_cost(2 * VOLATILE_REF_COST);
 9430   effect(TEMP_DEF res, KILL cr);
 9431   format %{
 9432     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9433   %}
 9434   ins_encode %{
 9435     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9436                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9437                /*weak*/ false, $res$$Register);
 9438     __ sxthw($res$$Register, $res$$Register);
 9439   %}
 9440   ins_pipe(pipe_slow);
 9441 %}
 9442 
 9443 // This pattern is generated automatically from cas.m4.
 9444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9445 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9446 
 9447   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9448   ins_cost(2 * VOLATILE_REF_COST);
 9449   effect(TEMP_DEF res, KILL cr);
 9450   format %{
 9451     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9452   %}
 9453   ins_encode %{
 9454     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9455                Assembler::word, /*acquire*/ false, /*release*/ true,
 9456                /*weak*/ false, $res$$Register);
 9457   %}
 9458   ins_pipe(pipe_slow);
 9459 %}
 9460 
 9461 // This pattern is generated automatically from cas.m4.
 9462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9463 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9464 
 9465   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9466   ins_cost(2 * VOLATILE_REF_COST);
 9467   effect(TEMP_DEF res, KILL cr);
 9468   format %{
 9469     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9470   %}
 9471   ins_encode %{
 9472     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9473                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9474                /*weak*/ false, $res$$Register);
 9475   %}
 9476   ins_pipe(pipe_slow);
 9477 %}
 9478 
 9479 // This pattern is generated automatically from cas.m4.
 9480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9481 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9482 
 9483   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9484   ins_cost(2 * VOLATILE_REF_COST);
 9485   effect(TEMP_DEF res, KILL cr);
 9486   format %{
 9487     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9488   %}
 9489   ins_encode %{
 9490     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9491                Assembler::word, /*acquire*/ false, /*release*/ true,
 9492                /*weak*/ false, $res$$Register);
 9493   %}
 9494   ins_pipe(pipe_slow);
 9495 %}
 9496 
 9497 // This pattern is generated automatically from cas.m4.
 9498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9499 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9500   predicate(n->as_LoadStore()->barrier_data() == 0);
 9501   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9502   ins_cost(2 * VOLATILE_REF_COST);
 9503   effect(TEMP_DEF res, KILL cr);
 9504   format %{
 9505     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9506   %}
 9507   ins_encode %{
 9508     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9509                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9510                /*weak*/ false, $res$$Register);
 9511   %}
 9512   ins_pipe(pipe_slow);
 9513 %}
 9514 
 9515 // This pattern is generated automatically from cas.m4.
 9516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9517 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9518   predicate(needs_acquiring_load_exclusive(n));
 9519   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9520   ins_cost(VOLATILE_REF_COST);
 9521   effect(TEMP_DEF res, KILL cr);
 9522   format %{
 9523     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9524   %}
 9525   ins_encode %{
 9526     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9527                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9528                /*weak*/ false, $res$$Register);
 9529     __ sxtbw($res$$Register, $res$$Register);
 9530   %}
 9531   ins_pipe(pipe_slow);
 9532 %}
 9533 
 9534 // This pattern is generated automatically from cas.m4.
 9535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9536 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9537   predicate(needs_acquiring_load_exclusive(n));
 9538   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9539   ins_cost(VOLATILE_REF_COST);
 9540   effect(TEMP_DEF res, KILL cr);
 9541   format %{
 9542     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9543   %}
 9544   ins_encode %{
 9545     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9546                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9547                /*weak*/ false, $res$$Register);
 9548     __ sxthw($res$$Register, $res$$Register);
 9549   %}
 9550   ins_pipe(pipe_slow);
 9551 %}
 9552 
 9553 // This pattern is generated automatically from cas.m4.
 9554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9555 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9556   predicate(needs_acquiring_load_exclusive(n));
 9557   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9558   ins_cost(VOLATILE_REF_COST);
 9559   effect(TEMP_DEF res, KILL cr);
 9560   format %{
 9561     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9562   %}
 9563   ins_encode %{
 9564     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9565                Assembler::word, /*acquire*/ true, /*release*/ true,
 9566                /*weak*/ false, $res$$Register);
 9567   %}
 9568   ins_pipe(pipe_slow);
 9569 %}
 9570 
 9571 // This pattern is generated automatically from cas.m4.
 9572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9573 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9574   predicate(needs_acquiring_load_exclusive(n));
 9575   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9576   ins_cost(VOLATILE_REF_COST);
 9577   effect(TEMP_DEF res, KILL cr);
 9578   format %{
 9579     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9580   %}
 9581   ins_encode %{
 9582     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9583                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9584                /*weak*/ false, $res$$Register);
 9585   %}
 9586   ins_pipe(pipe_slow);
 9587 %}
 9588 
 9589 // This pattern is generated automatically from cas.m4.
 9590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9591 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9592   predicate(needs_acquiring_load_exclusive(n));
 9593   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9594   ins_cost(VOLATILE_REF_COST);
 9595   effect(TEMP_DEF res, KILL cr);
 9596   format %{
 9597     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9598   %}
 9599   ins_encode %{
 9600     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9601                Assembler::word, /*acquire*/ true, /*release*/ true,
 9602                /*weak*/ false, $res$$Register);
 9603   %}
 9604   ins_pipe(pipe_slow);
 9605 %}
 9606 
 9607 // This pattern is generated automatically from cas.m4.
 9608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9609 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9610   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9611   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9612   ins_cost(VOLATILE_REF_COST);
 9613   effect(TEMP_DEF res, KILL cr);
 9614   format %{
 9615     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9616   %}
 9617   ins_encode %{
 9618     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9619                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9620                /*weak*/ false, $res$$Register);
 9621   %}
 9622   ins_pipe(pipe_slow);
 9623 %}
 9624 
 9625 // This pattern is generated automatically from cas.m4.
 9626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9627 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9628 
 9629   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9630   ins_cost(2 * VOLATILE_REF_COST);
 9631   effect(KILL cr);
 9632   format %{
 9633     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9634     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9635   %}
 9636   ins_encode %{
 9637     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9638                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9639                /*weak*/ true, noreg);
 9640     __ csetw($res$$Register, Assembler::EQ);
 9641   %}
 9642   ins_pipe(pipe_slow);
 9643 %}
 9644 
 9645 // This pattern is generated automatically from cas.m4.
 9646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9647 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9648 
 9649   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9650   ins_cost(2 * VOLATILE_REF_COST);
 9651   effect(KILL cr);
 9652   format %{
 9653     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9654     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9655   %}
 9656   ins_encode %{
 9657     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9658                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9659                /*weak*/ true, noreg);
 9660     __ csetw($res$$Register, Assembler::EQ);
 9661   %}
 9662   ins_pipe(pipe_slow);
 9663 %}
 9664 
 9665 // This pattern is generated automatically from cas.m4.
 9666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9667 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9668 
 9669   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9670   ins_cost(2 * VOLATILE_REF_COST);
 9671   effect(KILL cr);
 9672   format %{
 9673     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9674     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9675   %}
 9676   ins_encode %{
 9677     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9678                Assembler::word, /*acquire*/ false, /*release*/ true,
 9679                /*weak*/ true, noreg);
 9680     __ csetw($res$$Register, Assembler::EQ);
 9681   %}
 9682   ins_pipe(pipe_slow);
 9683 %}
 9684 
 9685 // This pattern is generated automatically from cas.m4.
 9686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9687 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9688 
 9689   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9690   ins_cost(2 * VOLATILE_REF_COST);
 9691   effect(KILL cr);
 9692   format %{
 9693     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9694     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9695   %}
 9696   ins_encode %{
 9697     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9698                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9699                /*weak*/ true, noreg);
 9700     __ csetw($res$$Register, Assembler::EQ);
 9701   %}
 9702   ins_pipe(pipe_slow);
 9703 %}
 9704 
 9705 // This pattern is generated automatically from cas.m4.
 9706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9707 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9708 
 9709   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9710   ins_cost(2 * VOLATILE_REF_COST);
 9711   effect(KILL cr);
 9712   format %{
 9713     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9714     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9715   %}
 9716   ins_encode %{
 9717     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9718                Assembler::word, /*acquire*/ false, /*release*/ true,
 9719                /*weak*/ true, noreg);
 9720     __ csetw($res$$Register, Assembler::EQ);
 9721   %}
 9722   ins_pipe(pipe_slow);
 9723 %}
 9724 
 9725 // This pattern is generated automatically from cas.m4.
 9726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9727 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9728   predicate(n->as_LoadStore()->barrier_data() == 0);
 9729   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9730   ins_cost(2 * VOLATILE_REF_COST);
 9731   effect(KILL cr);
 9732   format %{
 9733     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9734     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9735   %}
 9736   ins_encode %{
 9737     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9738                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9739                /*weak*/ true, noreg);
 9740     __ csetw($res$$Register, Assembler::EQ);
 9741   %}
 9742   ins_pipe(pipe_slow);
 9743 %}
 9744 
 9745 // This pattern is generated automatically from cas.m4.
 9746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9747 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9748   predicate(needs_acquiring_load_exclusive(n));
 9749   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9750   ins_cost(VOLATILE_REF_COST);
 9751   effect(KILL cr);
 9752   format %{
 9753     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9754     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9755   %}
 9756   ins_encode %{
 9757     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9758                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9759                /*weak*/ true, noreg);
 9760     __ csetw($res$$Register, Assembler::EQ);
 9761   %}
 9762   ins_pipe(pipe_slow);
 9763 %}
 9764 
 9765 // This pattern is generated automatically from cas.m4.
 9766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9767 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9768   predicate(needs_acquiring_load_exclusive(n));
 9769   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9770   ins_cost(VOLATILE_REF_COST);
 9771   effect(KILL cr);
 9772   format %{
 9773     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9774     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9775   %}
 9776   ins_encode %{
 9777     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9778                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9779                /*weak*/ true, noreg);
 9780     __ csetw($res$$Register, Assembler::EQ);
 9781   %}
 9782   ins_pipe(pipe_slow);
 9783 %}
 9784 
 9785 // This pattern is generated automatically from cas.m4.
 9786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9787 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9788   predicate(needs_acquiring_load_exclusive(n));
 9789   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9790   ins_cost(VOLATILE_REF_COST);
 9791   effect(KILL cr);
 9792   format %{
 9793     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9794     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9795   %}
 9796   ins_encode %{
 9797     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9798                Assembler::word, /*acquire*/ true, /*release*/ true,
 9799                /*weak*/ true, noreg);
 9800     __ csetw($res$$Register, Assembler::EQ);
 9801   %}
 9802   ins_pipe(pipe_slow);
 9803 %}
 9804 
 9805 // This pattern is generated automatically from cas.m4.
 9806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9807 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9808   predicate(needs_acquiring_load_exclusive(n));
 9809   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9810   ins_cost(VOLATILE_REF_COST);
 9811   effect(KILL cr);
 9812   format %{
 9813     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9814     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9815   %}
 9816   ins_encode %{
 9817     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9818                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9819                /*weak*/ true, noreg);
 9820     __ csetw($res$$Register, Assembler::EQ);
 9821   %}
 9822   ins_pipe(pipe_slow);
 9823 %}
 9824 
 9825 // This pattern is generated automatically from cas.m4.
 9826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9827 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9828   predicate(needs_acquiring_load_exclusive(n));
 9829   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9830   ins_cost(VOLATILE_REF_COST);
 9831   effect(KILL cr);
 9832   format %{
 9833     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9834     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9835   %}
 9836   ins_encode %{
 9837     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9838                Assembler::word, /*acquire*/ true, /*release*/ true,
 9839                /*weak*/ true, noreg);
 9840     __ csetw($res$$Register, Assembler::EQ);
 9841   %}
 9842   ins_pipe(pipe_slow);
 9843 %}
 9844 
 9845 // This pattern is generated automatically from cas.m4.
 9846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9847 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9848   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9849   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9850   ins_cost(VOLATILE_REF_COST);
 9851   effect(KILL cr);
 9852   format %{
 9853     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9854     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9855   %}
 9856   ins_encode %{
 9857     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9858                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9859                /*weak*/ true, noreg);
 9860     __ csetw($res$$Register, Assembler::EQ);
 9861   %}
 9862   ins_pipe(pipe_slow);
 9863 %}
 9864 
 9865 // END This section of the file is automatically generated. Do not edit --------------
 9866 // ---------------------------------------------------------------------
 9867 
 9868 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9869   match(Set prev (GetAndSetI mem newv));
 9870   ins_cost(2 * VOLATILE_REF_COST);
 9871   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9872   ins_encode %{
 9873     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9874   %}
 9875   ins_pipe(pipe_serial);
 9876 %}
 9877 
 9878 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9879   match(Set prev (GetAndSetL mem newv));
 9880   ins_cost(2 * VOLATILE_REF_COST);
 9881   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9882   ins_encode %{
 9883     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9884   %}
 9885   ins_pipe(pipe_serial);
 9886 %}
 9887 
 9888 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9889   match(Set prev (GetAndSetN mem newv));
 9890   ins_cost(2 * VOLATILE_REF_COST);
 9891   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9892   ins_encode %{
 9893     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9894   %}
 9895   ins_pipe(pipe_serial);
 9896 %}
 9897 
 9898 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9899   predicate(n->as_LoadStore()->barrier_data() == 0);
 9900   match(Set prev (GetAndSetP mem newv));
 9901   ins_cost(2 * VOLATILE_REF_COST);
 9902   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9903   ins_encode %{
 9904     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9905   %}
 9906   ins_pipe(pipe_serial);
 9907 %}
 9908 
 9909 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9910   predicate(needs_acquiring_load_exclusive(n));
 9911   match(Set prev (GetAndSetI mem newv));
 9912   ins_cost(VOLATILE_REF_COST);
 9913   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9914   ins_encode %{
 9915     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9916   %}
 9917   ins_pipe(pipe_serial);
 9918 %}
 9919 
 9920 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9921   predicate(needs_acquiring_load_exclusive(n));
 9922   match(Set prev (GetAndSetL mem newv));
 9923   ins_cost(VOLATILE_REF_COST);
 9924   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9925   ins_encode %{
 9926     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9927   %}
 9928   ins_pipe(pipe_serial);
 9929 %}
 9930 
 9931 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9932   predicate(needs_acquiring_load_exclusive(n));
 9933   match(Set prev (GetAndSetN mem newv));
 9934   ins_cost(VOLATILE_REF_COST);
 9935   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9936   ins_encode %{
 9937     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9938   %}
 9939   ins_pipe(pipe_serial);
 9940 %}
 9941 
 9942 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9943   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9944   match(Set prev (GetAndSetP mem newv));
 9945   ins_cost(VOLATILE_REF_COST);
 9946   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9947   ins_encode %{
 9948     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9949   %}
 9950   ins_pipe(pipe_serial);
 9951 %}
 9952 
 9953 
 9954 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9955   match(Set newval (GetAndAddL mem incr));
 9956   ins_cost(2 * VOLATILE_REF_COST + 1);
 9957   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9958   ins_encode %{
 9959     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9960   %}
 9961   ins_pipe(pipe_serial);
 9962 %}
 9963 
 9964 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9965   predicate(n->as_LoadStore()->result_not_used());
 9966   match(Set dummy (GetAndAddL mem incr));
 9967   ins_cost(2 * VOLATILE_REF_COST);
 9968   format %{ "get_and_addL [$mem], $incr" %}
 9969   ins_encode %{
 9970     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9971   %}
 9972   ins_pipe(pipe_serial);
 9973 %}
 9974 
 9975 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9976   match(Set newval (GetAndAddL mem incr));
 9977   ins_cost(2 * VOLATILE_REF_COST + 1);
 9978   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9979   ins_encode %{
 9980     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9981   %}
 9982   ins_pipe(pipe_serial);
 9983 %}
 9984 
 9985 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9986   predicate(n->as_LoadStore()->result_not_used());
 9987   match(Set dummy (GetAndAddL mem incr));
 9988   ins_cost(2 * VOLATILE_REF_COST);
 9989   format %{ "get_and_addL [$mem], $incr" %}
 9990   ins_encode %{
 9991     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9992   %}
 9993   ins_pipe(pipe_serial);
 9994 %}
 9995 
 9996 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9997   match(Set newval (GetAndAddI mem incr));
 9998   ins_cost(2 * VOLATILE_REF_COST + 1);
 9999   format %{ "get_and_addI $newval, [$mem], $incr" %}
10000   ins_encode %{
10001     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10002   %}
10003   ins_pipe(pipe_serial);
10004 %}
10005 
10006 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10007   predicate(n->as_LoadStore()->result_not_used());
10008   match(Set dummy (GetAndAddI mem incr));
10009   ins_cost(2 * VOLATILE_REF_COST);
10010   format %{ "get_and_addI [$mem], $incr" %}
10011   ins_encode %{
10012     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10013   %}
10014   ins_pipe(pipe_serial);
10015 %}
10016 
10017 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10018   match(Set newval (GetAndAddI mem incr));
10019   ins_cost(2 * VOLATILE_REF_COST + 1);
10020   format %{ "get_and_addI $newval, [$mem], $incr" %}
10021   ins_encode %{
10022     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10023   %}
10024   ins_pipe(pipe_serial);
10025 %}
10026 
10027 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10028   predicate(n->as_LoadStore()->result_not_used());
10029   match(Set dummy (GetAndAddI mem incr));
10030   ins_cost(2 * VOLATILE_REF_COST);
10031   format %{ "get_and_addI [$mem], $incr" %}
10032   ins_encode %{
10033     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10034   %}
10035   ins_pipe(pipe_serial);
10036 %}
10037 
10038 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10039   predicate(needs_acquiring_load_exclusive(n));
10040   match(Set newval (GetAndAddL mem incr));
10041   ins_cost(VOLATILE_REF_COST + 1);
10042   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10043   ins_encode %{
10044     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10045   %}
10046   ins_pipe(pipe_serial);
10047 %}
10048 
10049 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10050   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10051   match(Set dummy (GetAndAddL mem incr));
10052   ins_cost(VOLATILE_REF_COST);
10053   format %{ "get_and_addL_acq [$mem], $incr" %}
10054   ins_encode %{
10055     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10056   %}
10057   ins_pipe(pipe_serial);
10058 %}
10059 
10060 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10061   predicate(needs_acquiring_load_exclusive(n));
10062   match(Set newval (GetAndAddL mem incr));
10063   ins_cost(VOLATILE_REF_COST + 1);
10064   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10065   ins_encode %{
10066     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10067   %}
10068   ins_pipe(pipe_serial);
10069 %}
10070 
10071 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10072   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10073   match(Set dummy (GetAndAddL mem incr));
10074   ins_cost(VOLATILE_REF_COST);
10075   format %{ "get_and_addL_acq [$mem], $incr" %}
10076   ins_encode %{
10077     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10078   %}
10079   ins_pipe(pipe_serial);
10080 %}
10081 
10082 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10083   predicate(needs_acquiring_load_exclusive(n));
10084   match(Set newval (GetAndAddI mem incr));
10085   ins_cost(VOLATILE_REF_COST + 1);
10086   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10087   ins_encode %{
10088     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10089   %}
10090   ins_pipe(pipe_serial);
10091 %}
10092 
10093 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10094   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10095   match(Set dummy (GetAndAddI mem incr));
10096   ins_cost(VOLATILE_REF_COST);
10097   format %{ "get_and_addI_acq [$mem], $incr" %}
10098   ins_encode %{
10099     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10100   %}
10101   ins_pipe(pipe_serial);
10102 %}
10103 
10104 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10105   predicate(needs_acquiring_load_exclusive(n));
10106   match(Set newval (GetAndAddI mem incr));
10107   ins_cost(VOLATILE_REF_COST + 1);
10108   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10109   ins_encode %{
10110     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10111   %}
10112   ins_pipe(pipe_serial);
10113 %}
10114 
10115 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10116   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10117   match(Set dummy (GetAndAddI mem incr));
10118   ins_cost(VOLATILE_REF_COST);
10119   format %{ "get_and_addI_acq [$mem], $incr" %}
10120   ins_encode %{
10121     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10122   %}
10123   ins_pipe(pipe_serial);
10124 %}
10125 
10126 // Manifest a CmpL result in an integer register.
10127 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10128 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10129 %{
10130   match(Set dst (CmpL3 src1 src2));
10131   effect(KILL flags);
10132 
10133   ins_cost(INSN_COST * 6);
10134   format %{
10135       "cmp $src1, $src2"
10136       "csetw $dst, ne"
10137       "cnegw $dst, lt"
10138   %}
10139   // format %{ "CmpL3 $dst, $src1, $src2" %}
10140   ins_encode %{
10141     __ cmp($src1$$Register, $src2$$Register);
10142     __ csetw($dst$$Register, Assembler::NE);
10143     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10144   %}
10145 
10146   ins_pipe(pipe_class_default);
10147 %}
10148 
10149 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10150 %{
10151   match(Set dst (CmpL3 src1 src2));
10152   effect(KILL flags);
10153 
10154   ins_cost(INSN_COST * 6);
10155   format %{
10156       "cmp $src1, $src2"
10157       "csetw $dst, ne"
10158       "cnegw $dst, lt"
10159   %}
10160   ins_encode %{
10161     int32_t con = (int32_t)$src2$$constant;
10162      if (con < 0) {
10163       __ adds(zr, $src1$$Register, -con);
10164     } else {
10165       __ subs(zr, $src1$$Register, con);
10166     }
10167     __ csetw($dst$$Register, Assembler::NE);
10168     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10169   %}
10170 
10171   ins_pipe(pipe_class_default);
10172 %}
10173 
10174 // ============================================================================
10175 // Conditional Move Instructions
10176 
10177 // n.b. we have identical rules for both a signed compare op (cmpOp)
10178 // and an unsigned compare op (cmpOpU). it would be nice if we could
10179 // define an op class which merged both inputs and use it to type the
10180 // argument to a single rule. unfortunatelyt his fails because the
10181 // opclass does not live up to the COND_INTER interface of its
10182 // component operands. When the generic code tries to negate the
10183 // operand it ends up running the generci Machoper::negate method
10184 // which throws a ShouldNotHappen. So, we have to provide two flavours
10185 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10186 
10187 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10188   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10189 
10190   ins_cost(INSN_COST * 2);
10191   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10192 
10193   ins_encode %{
10194     __ cselw(as_Register($dst$$reg),
10195              as_Register($src2$$reg),
10196              as_Register($src1$$reg),
10197              (Assembler::Condition)$cmp$$cmpcode);
10198   %}
10199 
10200   ins_pipe(icond_reg_reg);
10201 %}
10202 
10203 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10204   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10205 
10206   ins_cost(INSN_COST * 2);
10207   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10208 
10209   ins_encode %{
10210     __ cselw(as_Register($dst$$reg),
10211              as_Register($src2$$reg),
10212              as_Register($src1$$reg),
10213              (Assembler::Condition)$cmp$$cmpcode);
10214   %}
10215 
10216   ins_pipe(icond_reg_reg);
10217 %}
10218 
10219 // special cases where one arg is zero
10220 
10221 // n.b. this is selected in preference to the rule above because it
10222 // avoids loading constant 0 into a source register
10223 
10224 // TODO
10225 // we ought only to be able to cull one of these variants as the ideal
10226 // transforms ought always to order the zero consistently (to left/right?)
10227 
10228 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10229   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10230 
10231   ins_cost(INSN_COST * 2);
10232   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10233 
10234   ins_encode %{
10235     __ cselw(as_Register($dst$$reg),
10236              as_Register($src$$reg),
10237              zr,
10238              (Assembler::Condition)$cmp$$cmpcode);
10239   %}
10240 
10241   ins_pipe(icond_reg);
10242 %}
10243 
10244 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10245   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10246 
10247   ins_cost(INSN_COST * 2);
10248   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10249 
10250   ins_encode %{
10251     __ cselw(as_Register($dst$$reg),
10252              as_Register($src$$reg),
10253              zr,
10254              (Assembler::Condition)$cmp$$cmpcode);
10255   %}
10256 
10257   ins_pipe(icond_reg);
10258 %}
10259 
10260 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10261   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10262 
10263   ins_cost(INSN_COST * 2);
10264   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10265 
10266   ins_encode %{
10267     __ cselw(as_Register($dst$$reg),
10268              zr,
10269              as_Register($src$$reg),
10270              (Assembler::Condition)$cmp$$cmpcode);
10271   %}
10272 
10273   ins_pipe(icond_reg);
10274 %}
10275 
10276 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10277   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10278 
10279   ins_cost(INSN_COST * 2);
10280   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10281 
10282   ins_encode %{
10283     __ cselw(as_Register($dst$$reg),
10284              zr,
10285              as_Register($src$$reg),
10286              (Assembler::Condition)$cmp$$cmpcode);
10287   %}
10288 
10289   ins_pipe(icond_reg);
10290 %}
10291 
10292 // special case for creating a boolean 0 or 1
10293 
10294 // n.b. this is selected in preference to the rule above because it
10295 // avoids loading constants 0 and 1 into a source register
10296 
10297 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10298   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10299 
10300   ins_cost(INSN_COST * 2);
10301   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10302 
10303   ins_encode %{
10304     // equivalently
10305     // cset(as_Register($dst$$reg),
10306     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10307     __ csincw(as_Register($dst$$reg),
10308              zr,
10309              zr,
10310              (Assembler::Condition)$cmp$$cmpcode);
10311   %}
10312 
10313   ins_pipe(icond_none);
10314 %}
10315 
10316 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10317   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10321 
10322   ins_encode %{
10323     // equivalently
10324     // cset(as_Register($dst$$reg),
10325     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10326     __ csincw(as_Register($dst$$reg),
10327              zr,
10328              zr,
10329              (Assembler::Condition)$cmp$$cmpcode);
10330   %}
10331 
10332   ins_pipe(icond_none);
10333 %}
10334 
10335 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10336   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10337 
10338   ins_cost(INSN_COST * 2);
10339   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10340 
10341   ins_encode %{
10342     __ csel(as_Register($dst$$reg),
10343             as_Register($src2$$reg),
10344             as_Register($src1$$reg),
10345             (Assembler::Condition)$cmp$$cmpcode);
10346   %}
10347 
10348   ins_pipe(icond_reg_reg);
10349 %}
10350 
10351 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10352   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10353 
10354   ins_cost(INSN_COST * 2);
10355   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10356 
10357   ins_encode %{
10358     __ csel(as_Register($dst$$reg),
10359             as_Register($src2$$reg),
10360             as_Register($src1$$reg),
10361             (Assembler::Condition)$cmp$$cmpcode);
10362   %}
10363 
10364   ins_pipe(icond_reg_reg);
10365 %}
10366 
10367 // special cases where one arg is zero
10368 
10369 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10370   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10371 
10372   ins_cost(INSN_COST * 2);
10373   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10374 
10375   ins_encode %{
10376     __ csel(as_Register($dst$$reg),
10377             zr,
10378             as_Register($src$$reg),
10379             (Assembler::Condition)$cmp$$cmpcode);
10380   %}
10381 
10382   ins_pipe(icond_reg);
10383 %}
10384 
10385 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10386   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10387 
10388   ins_cost(INSN_COST * 2);
10389   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10390 
10391   ins_encode %{
10392     __ csel(as_Register($dst$$reg),
10393             zr,
10394             as_Register($src$$reg),
10395             (Assembler::Condition)$cmp$$cmpcode);
10396   %}
10397 
10398   ins_pipe(icond_reg);
10399 %}
10400 
10401 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10402   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10403 
10404   ins_cost(INSN_COST * 2);
10405   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10406 
10407   ins_encode %{
10408     __ csel(as_Register($dst$$reg),
10409             as_Register($src$$reg),
10410             zr,
10411             (Assembler::Condition)$cmp$$cmpcode);
10412   %}
10413 
10414   ins_pipe(icond_reg);
10415 %}
10416 
10417 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10418   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10419 
10420   ins_cost(INSN_COST * 2);
10421   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10422 
10423   ins_encode %{
10424     __ csel(as_Register($dst$$reg),
10425             as_Register($src$$reg),
10426             zr,
10427             (Assembler::Condition)$cmp$$cmpcode);
10428   %}
10429 
10430   ins_pipe(icond_reg);
10431 %}
10432 
10433 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10434   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10435 
10436   ins_cost(INSN_COST * 2);
10437   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10438 
10439   ins_encode %{
10440     __ csel(as_Register($dst$$reg),
10441             as_Register($src2$$reg),
10442             as_Register($src1$$reg),
10443             (Assembler::Condition)$cmp$$cmpcode);
10444   %}
10445 
10446   ins_pipe(icond_reg_reg);
10447 %}
10448 
10449 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10450   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10451 
10452   ins_cost(INSN_COST * 2);
10453   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10454 
10455   ins_encode %{
10456     __ csel(as_Register($dst$$reg),
10457             as_Register($src2$$reg),
10458             as_Register($src1$$reg),
10459             (Assembler::Condition)$cmp$$cmpcode);
10460   %}
10461 
10462   ins_pipe(icond_reg_reg);
10463 %}
10464 
10465 // special cases where one arg is zero
10466 
10467 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10468   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10469 
10470   ins_cost(INSN_COST * 2);
10471   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10472 
10473   ins_encode %{
10474     __ csel(as_Register($dst$$reg),
10475             zr,
10476             as_Register($src$$reg),
10477             (Assembler::Condition)$cmp$$cmpcode);
10478   %}
10479 
10480   ins_pipe(icond_reg);
10481 %}
10482 
10483 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10484   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10485 
10486   ins_cost(INSN_COST * 2);
10487   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10488 
10489   ins_encode %{
10490     __ csel(as_Register($dst$$reg),
10491             zr,
10492             as_Register($src$$reg),
10493             (Assembler::Condition)$cmp$$cmpcode);
10494   %}
10495 
10496   ins_pipe(icond_reg);
10497 %}
10498 
10499 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10500   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10501 
10502   ins_cost(INSN_COST * 2);
10503   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10504 
10505   ins_encode %{
10506     __ csel(as_Register($dst$$reg),
10507             as_Register($src$$reg),
10508             zr,
10509             (Assembler::Condition)$cmp$$cmpcode);
10510   %}
10511 
10512   ins_pipe(icond_reg);
10513 %}
10514 
10515 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10516   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10517 
10518   ins_cost(INSN_COST * 2);
10519   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10520 
10521   ins_encode %{
10522     __ csel(as_Register($dst$$reg),
10523             as_Register($src$$reg),
10524             zr,
10525             (Assembler::Condition)$cmp$$cmpcode);
10526   %}
10527 
10528   ins_pipe(icond_reg);
10529 %}
10530 
10531 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10532   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10533 
10534   ins_cost(INSN_COST * 2);
10535   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10536 
10537   ins_encode %{
10538     __ cselw(as_Register($dst$$reg),
10539              as_Register($src2$$reg),
10540              as_Register($src1$$reg),
10541              (Assembler::Condition)$cmp$$cmpcode);
10542   %}
10543 
10544   ins_pipe(icond_reg_reg);
10545 %}
10546 
10547 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10548   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10549 
10550   ins_cost(INSN_COST * 2);
10551   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10552 
10553   ins_encode %{
10554     __ cselw(as_Register($dst$$reg),
10555              as_Register($src2$$reg),
10556              as_Register($src1$$reg),
10557              (Assembler::Condition)$cmp$$cmpcode);
10558   %}
10559 
10560   ins_pipe(icond_reg_reg);
10561 %}
10562 
10563 // special cases where one arg is zero
10564 
10565 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10566   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10567 
10568   ins_cost(INSN_COST * 2);
10569   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10570 
10571   ins_encode %{
10572     __ cselw(as_Register($dst$$reg),
10573              zr,
10574              as_Register($src$$reg),
10575              (Assembler::Condition)$cmp$$cmpcode);
10576   %}
10577 
10578   ins_pipe(icond_reg);
10579 %}
10580 
10581 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10582   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10583 
10584   ins_cost(INSN_COST * 2);
10585   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10586 
10587   ins_encode %{
10588     __ cselw(as_Register($dst$$reg),
10589              zr,
10590              as_Register($src$$reg),
10591              (Assembler::Condition)$cmp$$cmpcode);
10592   %}
10593 
10594   ins_pipe(icond_reg);
10595 %}
10596 
10597 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10598   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10599 
10600   ins_cost(INSN_COST * 2);
10601   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10602 
10603   ins_encode %{
10604     __ cselw(as_Register($dst$$reg),
10605              as_Register($src$$reg),
10606              zr,
10607              (Assembler::Condition)$cmp$$cmpcode);
10608   %}
10609 
10610   ins_pipe(icond_reg);
10611 %}
10612 
10613 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10614   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10615 
10616   ins_cost(INSN_COST * 2);
10617   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10618 
10619   ins_encode %{
10620     __ cselw(as_Register($dst$$reg),
10621              as_Register($src$$reg),
10622              zr,
10623              (Assembler::Condition)$cmp$$cmpcode);
10624   %}
10625 
10626   ins_pipe(icond_reg);
10627 %}
10628 
10629 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10630 %{
10631   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10632 
10633   ins_cost(INSN_COST * 3);
10634 
10635   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10636   ins_encode %{
10637     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10638     __ fcsels(as_FloatRegister($dst$$reg),
10639               as_FloatRegister($src2$$reg),
10640               as_FloatRegister($src1$$reg),
10641               cond);
10642   %}
10643 
10644   ins_pipe(fp_cond_reg_reg_s);
10645 %}
10646 
10647 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10648 %{
10649   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10650 
10651   ins_cost(INSN_COST * 3);
10652 
10653   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10654   ins_encode %{
10655     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10656     __ fcsels(as_FloatRegister($dst$$reg),
10657               as_FloatRegister($src2$$reg),
10658               as_FloatRegister($src1$$reg),
10659               cond);
10660   %}
10661 
10662   ins_pipe(fp_cond_reg_reg_s);
10663 %}
10664 
10665 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10666 %{
10667   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10668 
10669   ins_cost(INSN_COST * 3);
10670 
10671   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10672   ins_encode %{
10673     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10674     __ fcseld(as_FloatRegister($dst$$reg),
10675               as_FloatRegister($src2$$reg),
10676               as_FloatRegister($src1$$reg),
10677               cond);
10678   %}
10679 
10680   ins_pipe(fp_cond_reg_reg_d);
10681 %}
10682 
10683 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10684 %{
10685   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10686 
10687   ins_cost(INSN_COST * 3);
10688 
10689   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10690   ins_encode %{
10691     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10692     __ fcseld(as_FloatRegister($dst$$reg),
10693               as_FloatRegister($src2$$reg),
10694               as_FloatRegister($src1$$reg),
10695               cond);
10696   %}
10697 
10698   ins_pipe(fp_cond_reg_reg_d);
10699 %}
10700 
10701 // ============================================================================
10702 // Arithmetic Instructions
10703 //
10704 
10705 // Integer Addition
10706 
10707 // TODO
10708 // these currently employ operations which do not set CR and hence are
10709 // not flagged as killing CR but we would like to isolate the cases
10710 // where we want to set flags from those where we don't. need to work
10711 // out how to do that.
10712 
10713 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10714   match(Set dst (AddI src1 src2));
10715 
10716   ins_cost(INSN_COST);
10717   format %{ "addw  $dst, $src1, $src2" %}
10718 
10719   ins_encode %{
10720     __ addw(as_Register($dst$$reg),
10721             as_Register($src1$$reg),
10722             as_Register($src2$$reg));
10723   %}
10724 
10725   ins_pipe(ialu_reg_reg);
10726 %}
10727 
10728 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10729   match(Set dst (AddI src1 src2));
10730 
10731   ins_cost(INSN_COST);
10732   format %{ "addw $dst, $src1, $src2" %}
10733 
10734   // use opcode to indicate that this is an add not a sub
10735   opcode(0x0);
10736 
10737   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10738 
10739   ins_pipe(ialu_reg_imm);
10740 %}
10741 
10742 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10743   match(Set dst (AddI (ConvL2I src1) src2));
10744 
10745   ins_cost(INSN_COST);
10746   format %{ "addw $dst, $src1, $src2" %}
10747 
10748   // use opcode to indicate that this is an add not a sub
10749   opcode(0x0);
10750 
10751   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10752 
10753   ins_pipe(ialu_reg_imm);
10754 %}
10755 
10756 // Pointer Addition
10757 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10758   match(Set dst (AddP src1 src2));
10759 
10760   ins_cost(INSN_COST);
10761   format %{ "add $dst, $src1, $src2\t# ptr" %}
10762 
10763   ins_encode %{
10764     __ add(as_Register($dst$$reg),
10765            as_Register($src1$$reg),
10766            as_Register($src2$$reg));
10767   %}
10768 
10769   ins_pipe(ialu_reg_reg);
10770 %}
10771 
10772 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10773   match(Set dst (AddP src1 (ConvI2L src2)));
10774 
10775   ins_cost(1.9 * INSN_COST);
10776   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10777 
10778   ins_encode %{
10779     __ add(as_Register($dst$$reg),
10780            as_Register($src1$$reg),
10781            as_Register($src2$$reg), ext::sxtw);
10782   %}
10783 
10784   ins_pipe(ialu_reg_reg);
10785 %}
10786 
10787 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10788   match(Set dst (AddP src1 (LShiftL src2 scale)));
10789 
10790   ins_cost(1.9 * INSN_COST);
10791   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10792 
10793   ins_encode %{
10794     __ lea(as_Register($dst$$reg),
10795            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10796                    Address::lsl($scale$$constant)));
10797   %}
10798 
10799   ins_pipe(ialu_reg_reg_shift);
10800 %}
10801 
10802 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10803   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10804 
10805   ins_cost(1.9 * INSN_COST);
10806   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10807 
10808   ins_encode %{
10809     __ lea(as_Register($dst$$reg),
10810            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10811                    Address::sxtw($scale$$constant)));
10812   %}
10813 
10814   ins_pipe(ialu_reg_reg_shift);
10815 %}
10816 
10817 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10818   match(Set dst (LShiftL (ConvI2L src) scale));
10819 
10820   ins_cost(INSN_COST);
10821   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10822 
10823   ins_encode %{
10824     __ sbfiz(as_Register($dst$$reg),
10825           as_Register($src$$reg),
10826           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10827   %}
10828 
10829   ins_pipe(ialu_reg_shift);
10830 %}
10831 
10832 // Pointer Immediate Addition
10833 // n.b. this needs to be more expensive than using an indirect memory
10834 // operand
10835 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10836   match(Set dst (AddP src1 src2));
10837 
10838   ins_cost(INSN_COST);
10839   format %{ "add $dst, $src1, $src2\t# ptr" %}
10840 
10841   // use opcode to indicate that this is an add not a sub
10842   opcode(0x0);
10843 
10844   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10845 
10846   ins_pipe(ialu_reg_imm);
10847 %}
10848 
10849 // Long Addition
10850 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10851 
10852   match(Set dst (AddL src1 src2));
10853 
10854   ins_cost(INSN_COST);
10855   format %{ "add  $dst, $src1, $src2" %}
10856 
10857   ins_encode %{
10858     __ add(as_Register($dst$$reg),
10859            as_Register($src1$$reg),
10860            as_Register($src2$$reg));
10861   %}
10862 
10863   ins_pipe(ialu_reg_reg);
10864 %}
10865 
10866 // No constant pool entries requiredLong Immediate Addition.
10867 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10868   match(Set dst (AddL src1 src2));
10869 
10870   ins_cost(INSN_COST);
10871   format %{ "add $dst, $src1, $src2" %}
10872 
10873   // use opcode to indicate that this is an add not a sub
10874   opcode(0x0);
10875 
10876   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10877 
10878   ins_pipe(ialu_reg_imm);
10879 %}
10880 
10881 // Integer Subtraction
10882 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10883   match(Set dst (SubI src1 src2));
10884 
10885   ins_cost(INSN_COST);
10886   format %{ "subw  $dst, $src1, $src2" %}
10887 
10888   ins_encode %{
10889     __ subw(as_Register($dst$$reg),
10890             as_Register($src1$$reg),
10891             as_Register($src2$$reg));
10892   %}
10893 
10894   ins_pipe(ialu_reg_reg);
10895 %}
10896 
10897 // Immediate Subtraction
10898 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10899   match(Set dst (SubI src1 src2));
10900 
10901   ins_cost(INSN_COST);
10902   format %{ "subw $dst, $src1, $src2" %}
10903 
10904   // use opcode to indicate that this is a sub not an add
10905   opcode(0x1);
10906 
10907   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10908 
10909   ins_pipe(ialu_reg_imm);
10910 %}
10911 
10912 // Long Subtraction
10913 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10914 
10915   match(Set dst (SubL src1 src2));
10916 
10917   ins_cost(INSN_COST);
10918   format %{ "sub  $dst, $src1, $src2" %}
10919 
10920   ins_encode %{
10921     __ sub(as_Register($dst$$reg),
10922            as_Register($src1$$reg),
10923            as_Register($src2$$reg));
10924   %}
10925 
10926   ins_pipe(ialu_reg_reg);
10927 %}
10928 
10929 // No constant pool entries requiredLong Immediate Subtraction.
10930 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10931   match(Set dst (SubL src1 src2));
10932 
10933   ins_cost(INSN_COST);
10934   format %{ "sub$dst, $src1, $src2" %}
10935 
10936   // use opcode to indicate that this is a sub not an add
10937   opcode(0x1);
10938 
10939   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10940 
10941   ins_pipe(ialu_reg_imm);
10942 %}
10943 
10944 // Integer Negation (special case for sub)
10945 
10946 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10947   match(Set dst (SubI zero src));
10948 
10949   ins_cost(INSN_COST);
10950   format %{ "negw $dst, $src\t# int" %}
10951 
10952   ins_encode %{
10953     __ negw(as_Register($dst$$reg),
10954             as_Register($src$$reg));
10955   %}
10956 
10957   ins_pipe(ialu_reg);
10958 %}
10959 
10960 // Long Negation
10961 
10962 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10963   match(Set dst (SubL zero src));
10964 
10965   ins_cost(INSN_COST);
10966   format %{ "neg $dst, $src\t# long" %}
10967 
10968   ins_encode %{
10969     __ neg(as_Register($dst$$reg),
10970            as_Register($src$$reg));
10971   %}
10972 
10973   ins_pipe(ialu_reg);
10974 %}
10975 
10976 // Integer Multiply
10977 
10978 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10979   match(Set dst (MulI src1 src2));
10980 
10981   ins_cost(INSN_COST * 3);
10982   format %{ "mulw  $dst, $src1, $src2" %}
10983 
10984   ins_encode %{
10985     __ mulw(as_Register($dst$$reg),
10986             as_Register($src1$$reg),
10987             as_Register($src2$$reg));
10988   %}
10989 
10990   ins_pipe(imul_reg_reg);
10991 %}
10992 
10993 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10994   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10995 
10996   ins_cost(INSN_COST * 3);
10997   format %{ "smull  $dst, $src1, $src2" %}
10998 
10999   ins_encode %{
11000     __ smull(as_Register($dst$$reg),
11001              as_Register($src1$$reg),
11002              as_Register($src2$$reg));
11003   %}
11004 
11005   ins_pipe(imul_reg_reg);
11006 %}
11007 
11008 // Long Multiply
11009 
11010 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11011   match(Set dst (MulL src1 src2));
11012 
11013   ins_cost(INSN_COST * 5);
11014   format %{ "mul  $dst, $src1, $src2" %}
11015 
11016   ins_encode %{
11017     __ mul(as_Register($dst$$reg),
11018            as_Register($src1$$reg),
11019            as_Register($src2$$reg));
11020   %}
11021 
11022   ins_pipe(lmul_reg_reg);
11023 %}
11024 
11025 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11026 %{
11027   match(Set dst (MulHiL src1 src2));
11028 
11029   ins_cost(INSN_COST * 7);
11030   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11031 
11032   ins_encode %{
11033     __ smulh(as_Register($dst$$reg),
11034              as_Register($src1$$reg),
11035              as_Register($src2$$reg));
11036   %}
11037 
11038   ins_pipe(lmul_reg_reg);
11039 %}
11040 
11041 // Combined Integer Multiply & Add/Sub
11042 
11043 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11044   match(Set dst (AddI src3 (MulI src1 src2)));
11045 
11046   ins_cost(INSN_COST * 3);
11047   format %{ "madd  $dst, $src1, $src2, $src3" %}
11048 
11049   ins_encode %{
11050     __ maddw(as_Register($dst$$reg),
11051              as_Register($src1$$reg),
11052              as_Register($src2$$reg),
11053              as_Register($src3$$reg));
11054   %}
11055 
11056   ins_pipe(imac_reg_reg);
11057 %}
11058 
11059 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11060   match(Set dst (SubI src3 (MulI src1 src2)));
11061 
11062   ins_cost(INSN_COST * 3);
11063   format %{ "msub  $dst, $src1, $src2, $src3" %}
11064 
11065   ins_encode %{
11066     __ msubw(as_Register($dst$$reg),
11067              as_Register($src1$$reg),
11068              as_Register($src2$$reg),
11069              as_Register($src3$$reg));
11070   %}
11071 
11072   ins_pipe(imac_reg_reg);
11073 %}
11074 
11075 // Combined Integer Multiply & Neg
11076 
11077 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11078   match(Set dst (MulI (SubI zero src1) src2));
11079 
11080   ins_cost(INSN_COST * 3);
11081   format %{ "mneg  $dst, $src1, $src2" %}
11082 
11083   ins_encode %{
11084     __ mnegw(as_Register($dst$$reg),
11085              as_Register($src1$$reg),
11086              as_Register($src2$$reg));
11087   %}
11088 
11089   ins_pipe(imac_reg_reg);
11090 %}
11091 
11092 // Combined Long Multiply & Add/Sub
11093 
11094 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11095   match(Set dst (AddL src3 (MulL src1 src2)));
11096 
11097   ins_cost(INSN_COST * 5);
11098   format %{ "madd  $dst, $src1, $src2, $src3" %}
11099 
11100   ins_encode %{
11101     __ madd(as_Register($dst$$reg),
11102             as_Register($src1$$reg),
11103             as_Register($src2$$reg),
11104             as_Register($src3$$reg));
11105   %}
11106 
11107   ins_pipe(lmac_reg_reg);
11108 %}
11109 
11110 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11111   match(Set dst (SubL src3 (MulL src1 src2)));
11112 
11113   ins_cost(INSN_COST * 5);
11114   format %{ "msub  $dst, $src1, $src2, $src3" %}
11115 
11116   ins_encode %{
11117     __ msub(as_Register($dst$$reg),
11118             as_Register($src1$$reg),
11119             as_Register($src2$$reg),
11120             as_Register($src3$$reg));
11121   %}
11122 
11123   ins_pipe(lmac_reg_reg);
11124 %}
11125 
11126 // Combined Long Multiply & Neg
11127 
11128 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11129   match(Set dst (MulL (SubL zero src1) src2));
11130 
11131   ins_cost(INSN_COST * 5);
11132   format %{ "mneg  $dst, $src1, $src2" %}
11133 
11134   ins_encode %{
11135     __ mneg(as_Register($dst$$reg),
11136             as_Register($src1$$reg),
11137             as_Register($src2$$reg));
11138   %}
11139 
11140   ins_pipe(lmac_reg_reg);
11141 %}
11142 
11143 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11144 
11145 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11146   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11147 
11148   ins_cost(INSN_COST * 3);
11149   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11150 
11151   ins_encode %{
11152     __ smaddl(as_Register($dst$$reg),
11153               as_Register($src1$$reg),
11154               as_Register($src2$$reg),
11155               as_Register($src3$$reg));
11156   %}
11157 
11158   ins_pipe(imac_reg_reg);
11159 %}
11160 
11161 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11162   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11163 
11164   ins_cost(INSN_COST * 3);
11165   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11166 
11167   ins_encode %{
11168     __ smsubl(as_Register($dst$$reg),
11169               as_Register($src1$$reg),
11170               as_Register($src2$$reg),
11171               as_Register($src3$$reg));
11172   %}
11173 
11174   ins_pipe(imac_reg_reg);
11175 %}
11176 
11177 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11178   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11179 
11180   ins_cost(INSN_COST * 3);
11181   format %{ "smnegl  $dst, $src1, $src2" %}
11182 
11183   ins_encode %{
11184     __ smnegl(as_Register($dst$$reg),
11185               as_Register($src1$$reg),
11186               as_Register($src2$$reg));
11187   %}
11188 
11189   ins_pipe(imac_reg_reg);
11190 %}
11191 
11192 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11193 
11194 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11195   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11196 
11197   ins_cost(INSN_COST * 5);
11198   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11199             "maddw $dst, $src3, $src4, rscratch1" %}
11200 
11201   ins_encode %{
11202     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11203     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11204 
11205   ins_pipe(imac_reg_reg);
11206 %}
11207 
11208 // Integer Divide
11209 
11210 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11211   match(Set dst (DivI src1 src2));
11212 
11213   ins_cost(INSN_COST * 19);
11214   format %{ "sdivw  $dst, $src1, $src2" %}
11215 
11216   ins_encode(aarch64_enc_divw(dst, src1, src2));
11217   ins_pipe(idiv_reg_reg);
11218 %}
11219 
11220 // Long Divide
11221 
11222 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11223   match(Set dst (DivL src1 src2));
11224 
11225   ins_cost(INSN_COST * 35);
11226   format %{ "sdiv   $dst, $src1, $src2" %}
11227 
11228   ins_encode(aarch64_enc_div(dst, src1, src2));
11229   ins_pipe(ldiv_reg_reg);
11230 %}
11231 
11232 // Integer Remainder
11233 
11234 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11235   match(Set dst (ModI src1 src2));
11236 
11237   ins_cost(INSN_COST * 22);
11238   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11239             "msubw($dst, rscratch1, $src2, $src1" %}
11240 
11241   ins_encode(aarch64_enc_modw(dst, src1, src2));
11242   ins_pipe(idiv_reg_reg);
11243 %}
11244 
11245 // Long Remainder
11246 
11247 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11248   match(Set dst (ModL src1 src2));
11249 
11250   ins_cost(INSN_COST * 38);
11251   format %{ "sdiv   rscratch1, $src1, $src2\n"
11252             "msub($dst, rscratch1, $src2, $src1" %}
11253 
11254   ins_encode(aarch64_enc_mod(dst, src1, src2));
11255   ins_pipe(ldiv_reg_reg);
11256 %}
11257 
11258 // Integer Shifts
11259 
11260 // Shift Left Register
11261 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11262   match(Set dst (LShiftI src1 src2));
11263 
11264   ins_cost(INSN_COST * 2);
11265   format %{ "lslvw  $dst, $src1, $src2" %}
11266 
11267   ins_encode %{
11268     __ lslvw(as_Register($dst$$reg),
11269              as_Register($src1$$reg),
11270              as_Register($src2$$reg));
11271   %}
11272 
11273   ins_pipe(ialu_reg_reg_vshift);
11274 %}
11275 
11276 // Shift Left Immediate
11277 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11278   match(Set dst (LShiftI src1 src2));
11279 
11280   ins_cost(INSN_COST);
11281   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11282 
11283   ins_encode %{
11284     __ lslw(as_Register($dst$$reg),
11285             as_Register($src1$$reg),
11286             $src2$$constant & 0x1f);
11287   %}
11288 
11289   ins_pipe(ialu_reg_shift);
11290 %}
11291 
11292 // Shift Right Logical Register
11293 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11294   match(Set dst (URShiftI src1 src2));
11295 
11296   ins_cost(INSN_COST * 2);
11297   format %{ "lsrvw  $dst, $src1, $src2" %}
11298 
11299   ins_encode %{
11300     __ lsrvw(as_Register($dst$$reg),
11301              as_Register($src1$$reg),
11302              as_Register($src2$$reg));
11303   %}
11304 
11305   ins_pipe(ialu_reg_reg_vshift);
11306 %}
11307 
11308 // Shift Right Logical Immediate
11309 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11310   match(Set dst (URShiftI src1 src2));
11311 
11312   ins_cost(INSN_COST);
11313   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11314 
11315   ins_encode %{
11316     __ lsrw(as_Register($dst$$reg),
11317             as_Register($src1$$reg),
11318             $src2$$constant & 0x1f);
11319   %}
11320 
11321   ins_pipe(ialu_reg_shift);
11322 %}
11323 
11324 // Shift Right Arithmetic Register
11325 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11326   match(Set dst (RShiftI src1 src2));
11327 
11328   ins_cost(INSN_COST * 2);
11329   format %{ "asrvw  $dst, $src1, $src2" %}
11330 
11331   ins_encode %{
11332     __ asrvw(as_Register($dst$$reg),
11333              as_Register($src1$$reg),
11334              as_Register($src2$$reg));
11335   %}
11336 
11337   ins_pipe(ialu_reg_reg_vshift);
11338 %}
11339 
11340 // Shift Right Arithmetic Immediate
11341 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11342   match(Set dst (RShiftI src1 src2));
11343 
11344   ins_cost(INSN_COST);
11345   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11346 
11347   ins_encode %{
11348     __ asrw(as_Register($dst$$reg),
11349             as_Register($src1$$reg),
11350             $src2$$constant & 0x1f);
11351   %}
11352 
11353   ins_pipe(ialu_reg_shift);
11354 %}
11355 
11356 // Combined Int Mask and Right Shift (using UBFM)
11357 // TODO
11358 
11359 // Long Shifts
11360 
11361 // Shift Left Register
11362 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11363   match(Set dst (LShiftL src1 src2));
11364 
11365   ins_cost(INSN_COST * 2);
11366   format %{ "lslv  $dst, $src1, $src2" %}
11367 
11368   ins_encode %{
11369     __ lslv(as_Register($dst$$reg),
11370             as_Register($src1$$reg),
11371             as_Register($src2$$reg));
11372   %}
11373 
11374   ins_pipe(ialu_reg_reg_vshift);
11375 %}
11376 
11377 // Shift Left Immediate
11378 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11379   match(Set dst (LShiftL src1 src2));
11380 
11381   ins_cost(INSN_COST);
11382   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11383 
11384   ins_encode %{
11385     __ lsl(as_Register($dst$$reg),
11386             as_Register($src1$$reg),
11387             $src2$$constant & 0x3f);
11388   %}
11389 
11390   ins_pipe(ialu_reg_shift);
11391 %}
11392 
11393 // Shift Right Logical Register
11394 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11395   match(Set dst (URShiftL src1 src2));
11396 
11397   ins_cost(INSN_COST * 2);
11398   format %{ "lsrv  $dst, $src1, $src2" %}
11399 
11400   ins_encode %{
11401     __ lsrv(as_Register($dst$$reg),
11402             as_Register($src1$$reg),
11403             as_Register($src2$$reg));
11404   %}
11405 
11406   ins_pipe(ialu_reg_reg_vshift);
11407 %}
11408 
11409 // Shift Right Logical Immediate
11410 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11411   match(Set dst (URShiftL src1 src2));
11412 
11413   ins_cost(INSN_COST);
11414   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11415 
11416   ins_encode %{
11417     __ lsr(as_Register($dst$$reg),
11418            as_Register($src1$$reg),
11419            $src2$$constant & 0x3f);
11420   %}
11421 
11422   ins_pipe(ialu_reg_shift);
11423 %}
11424 
11425 // A special-case pattern for card table stores.
11426 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11427   match(Set dst (URShiftL (CastP2X src1) src2));
11428 
11429   ins_cost(INSN_COST);
11430   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11431 
11432   ins_encode %{
11433     __ lsr(as_Register($dst$$reg),
11434            as_Register($src1$$reg),
11435            $src2$$constant & 0x3f);
11436   %}
11437 
11438   ins_pipe(ialu_reg_shift);
11439 %}
11440 
11441 // Shift Right Arithmetic Register
11442 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11443   match(Set dst (RShiftL src1 src2));
11444 
11445   ins_cost(INSN_COST * 2);
11446   format %{ "asrv  $dst, $src1, $src2" %}
11447 
11448   ins_encode %{
11449     __ asrv(as_Register($dst$$reg),
11450             as_Register($src1$$reg),
11451             as_Register($src2$$reg));
11452   %}
11453 
11454   ins_pipe(ialu_reg_reg_vshift);
11455 %}
11456 
11457 // Shift Right Arithmetic Immediate
11458 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11459   match(Set dst (RShiftL src1 src2));
11460 
11461   ins_cost(INSN_COST);
11462   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11463 
11464   ins_encode %{
11465     __ asr(as_Register($dst$$reg),
11466            as_Register($src1$$reg),
11467            $src2$$constant & 0x3f);
11468   %}
11469 
11470   ins_pipe(ialu_reg_shift);
11471 %}
11472 
11473 // BEGIN This section of the file is automatically generated. Do not edit --------------
11474 // This section is generated from aarch64_ad.m4
11475 
11476 
11477 // This pattern is automatically generated from aarch64_ad.m4
11478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11479 instruct regL_not_reg(iRegLNoSp dst,
11480                          iRegL src1, immL_M1 m1,
11481                          rFlagsReg cr) %{
11482   match(Set dst (XorL src1 m1));
11483   ins_cost(INSN_COST);
11484   format %{ "eon  $dst, $src1, zr" %}
11485 
11486   ins_encode %{
11487     __ eon(as_Register($dst$$reg),
11488               as_Register($src1$$reg),
11489               zr,
11490               Assembler::LSL, 0);
11491   %}
11492 
11493   ins_pipe(ialu_reg);
11494 %}
11495 
11496 // This pattern is automatically generated from aarch64_ad.m4
11497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11498 instruct regI_not_reg(iRegINoSp dst,
11499                          iRegIorL2I src1, immI_M1 m1,
11500                          rFlagsReg cr) %{
11501   match(Set dst (XorI src1 m1));
11502   ins_cost(INSN_COST);
11503   format %{ "eonw  $dst, $src1, zr" %}
11504 
11505   ins_encode %{
11506     __ eonw(as_Register($dst$$reg),
11507               as_Register($src1$$reg),
11508               zr,
11509               Assembler::LSL, 0);
11510   %}
11511 
11512   ins_pipe(ialu_reg);
11513 %}
11514 
11515 // This pattern is automatically generated from aarch64_ad.m4
11516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11517 instruct AndI_reg_not_reg(iRegINoSp dst,
11518                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11519   match(Set dst (AndI src1 (XorI src2 m1)));
11520   ins_cost(INSN_COST);
11521   format %{ "bicw  $dst, $src1, $src2" %}
11522 
11523   ins_encode %{
11524     __ bicw(as_Register($dst$$reg),
11525               as_Register($src1$$reg),
11526               as_Register($src2$$reg),
11527               Assembler::LSL, 0);
11528   %}
11529 
11530   ins_pipe(ialu_reg_reg);
11531 %}
11532 
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct AndL_reg_not_reg(iRegLNoSp dst,
11536                          iRegL src1, iRegL src2, immL_M1 m1) %{
11537   match(Set dst (AndL src1 (XorL src2 m1)));
11538   ins_cost(INSN_COST);
11539   format %{ "bic  $dst, $src1, $src2" %}
11540 
11541   ins_encode %{
11542     __ bic(as_Register($dst$$reg),
11543               as_Register($src1$$reg),
11544               as_Register($src2$$reg),
11545               Assembler::LSL, 0);
11546   %}
11547 
11548   ins_pipe(ialu_reg_reg);
11549 %}
11550 
11551 // This pattern is automatically generated from aarch64_ad.m4
11552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11553 instruct OrI_reg_not_reg(iRegINoSp dst,
11554                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11555   match(Set dst (OrI src1 (XorI src2 m1)));
11556   ins_cost(INSN_COST);
11557   format %{ "ornw  $dst, $src1, $src2" %}
11558 
11559   ins_encode %{
11560     __ ornw(as_Register($dst$$reg),
11561               as_Register($src1$$reg),
11562               as_Register($src2$$reg),
11563               Assembler::LSL, 0);
11564   %}
11565 
11566   ins_pipe(ialu_reg_reg);
11567 %}
11568 
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571 instruct OrL_reg_not_reg(iRegLNoSp dst,
11572                          iRegL src1, iRegL src2, immL_M1 m1) %{
11573   match(Set dst (OrL src1 (XorL src2 m1)));
11574   ins_cost(INSN_COST);
11575   format %{ "orn  $dst, $src1, $src2" %}
11576 
11577   ins_encode %{
11578     __ orn(as_Register($dst$$reg),
11579               as_Register($src1$$reg),
11580               as_Register($src2$$reg),
11581               Assembler::LSL, 0);
11582   %}
11583 
11584   ins_pipe(ialu_reg_reg);
11585 %}
11586 
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 instruct XorI_reg_not_reg(iRegINoSp dst,
11590                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11591   match(Set dst (XorI m1 (XorI src2 src1)));
11592   ins_cost(INSN_COST);
11593   format %{ "eonw  $dst, $src1, $src2" %}
11594 
11595   ins_encode %{
11596     __ eonw(as_Register($dst$$reg),
11597               as_Register($src1$$reg),
11598               as_Register($src2$$reg),
11599               Assembler::LSL, 0);
11600   %}
11601 
11602   ins_pipe(ialu_reg_reg);
11603 %}
11604 
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607 instruct XorL_reg_not_reg(iRegLNoSp dst,
11608                          iRegL src1, iRegL src2, immL_M1 m1) %{
11609   match(Set dst (XorL m1 (XorL src2 src1)));
11610   ins_cost(INSN_COST);
11611   format %{ "eon  $dst, $src1, $src2" %}
11612 
11613   ins_encode %{
11614     __ eon(as_Register($dst$$reg),
11615               as_Register($src1$$reg),
11616               as_Register($src2$$reg),
11617               Assembler::LSL, 0);
11618   %}
11619 
11620   ins_pipe(ialu_reg_reg);
11621 %}
11622 
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625 // val & (-1 ^ (val >>> shift)) ==> bicw
11626 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11627                          iRegIorL2I src1, iRegIorL2I src2,
11628                          immI src3, immI_M1 src4) %{
11629   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11630   ins_cost(1.9 * INSN_COST);
11631   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11632 
11633   ins_encode %{
11634     __ bicw(as_Register($dst$$reg),
11635               as_Register($src1$$reg),
11636               as_Register($src2$$reg),
11637               Assembler::LSR,
11638               $src3$$constant & 0x1f);
11639   %}
11640 
11641   ins_pipe(ialu_reg_reg_shift);
11642 %}
11643 
11644 // This pattern is automatically generated from aarch64_ad.m4
11645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11646 // val & (-1 ^ (val >>> shift)) ==> bic
11647 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11648                          iRegL src1, iRegL src2,
11649                          immI src3, immL_M1 src4) %{
11650   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11651   ins_cost(1.9 * INSN_COST);
11652   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11653 
11654   ins_encode %{
11655     __ bic(as_Register($dst$$reg),
11656               as_Register($src1$$reg),
11657               as_Register($src2$$reg),
11658               Assembler::LSR,
11659               $src3$$constant & 0x3f);
11660   %}
11661 
11662   ins_pipe(ialu_reg_reg_shift);
11663 %}
11664 
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667 // val & (-1 ^ (val >> shift)) ==> bicw
11668 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11669                          iRegIorL2I src1, iRegIorL2I src2,
11670                          immI src3, immI_M1 src4) %{
11671   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11672   ins_cost(1.9 * INSN_COST);
11673   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11674 
11675   ins_encode %{
11676     __ bicw(as_Register($dst$$reg),
11677               as_Register($src1$$reg),
11678               as_Register($src2$$reg),
11679               Assembler::ASR,
11680               $src3$$constant & 0x1f);
11681   %}
11682 
11683   ins_pipe(ialu_reg_reg_shift);
11684 %}
11685 
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 // val & (-1 ^ (val >> shift)) ==> bic
11689 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11690                          iRegL src1, iRegL src2,
11691                          immI src3, immL_M1 src4) %{
11692   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11693   ins_cost(1.9 * INSN_COST);
11694   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11695 
11696   ins_encode %{
11697     __ bic(as_Register($dst$$reg),
11698               as_Register($src1$$reg),
11699               as_Register($src2$$reg),
11700               Assembler::ASR,
11701               $src3$$constant & 0x3f);
11702   %}
11703 
11704   ins_pipe(ialu_reg_reg_shift);
11705 %}
11706 
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 // val & (-1 ^ (val ror shift)) ==> bicw
11710 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11711                          iRegIorL2I src1, iRegIorL2I src2,
11712                          immI src3, immI_M1 src4) %{
11713   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11714   ins_cost(1.9 * INSN_COST);
11715   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11716 
11717   ins_encode %{
11718     __ bicw(as_Register($dst$$reg),
11719               as_Register($src1$$reg),
11720               as_Register($src2$$reg),
11721               Assembler::ROR,
11722               $src3$$constant & 0x1f);
11723   %}
11724 
11725   ins_pipe(ialu_reg_reg_shift);
11726 %}
11727 
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 // val & (-1 ^ (val ror shift)) ==> bic
11731 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11732                          iRegL src1, iRegL src2,
11733                          immI src3, immL_M1 src4) %{
11734   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11735   ins_cost(1.9 * INSN_COST);
11736   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11737 
11738   ins_encode %{
11739     __ bic(as_Register($dst$$reg),
11740               as_Register($src1$$reg),
11741               as_Register($src2$$reg),
11742               Assembler::ROR,
11743               $src3$$constant & 0x3f);
11744   %}
11745 
11746   ins_pipe(ialu_reg_reg_shift);
11747 %}
11748 
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 // val & (-1 ^ (val << shift)) ==> bicw
11752 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11753                          iRegIorL2I src1, iRegIorL2I src2,
11754                          immI src3, immI_M1 src4) %{
11755   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11756   ins_cost(1.9 * INSN_COST);
11757   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11758 
11759   ins_encode %{
11760     __ bicw(as_Register($dst$$reg),
11761               as_Register($src1$$reg),
11762               as_Register($src2$$reg),
11763               Assembler::LSL,
11764               $src3$$constant & 0x1f);
11765   %}
11766 
11767   ins_pipe(ialu_reg_reg_shift);
11768 %}
11769 
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 // val & (-1 ^ (val << shift)) ==> bic
11773 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11774                          iRegL src1, iRegL src2,
11775                          immI src3, immL_M1 src4) %{
11776   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11777   ins_cost(1.9 * INSN_COST);
11778   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11779 
11780   ins_encode %{
11781     __ bic(as_Register($dst$$reg),
11782               as_Register($src1$$reg),
11783               as_Register($src2$$reg),
11784               Assembler::LSL,
11785               $src3$$constant & 0x3f);
11786   %}
11787 
11788   ins_pipe(ialu_reg_reg_shift);
11789 %}
11790 
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11794 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11795                          iRegIorL2I src1, iRegIorL2I src2,
11796                          immI src3, immI_M1 src4) %{
11797   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11798   ins_cost(1.9 * INSN_COST);
11799   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11800 
11801   ins_encode %{
11802     __ eonw(as_Register($dst$$reg),
11803               as_Register($src1$$reg),
11804               as_Register($src2$$reg),
11805               Assembler::LSR,
11806               $src3$$constant & 0x1f);
11807   %}
11808 
11809   ins_pipe(ialu_reg_reg_shift);
11810 %}
11811 
11812 // This pattern is automatically generated from aarch64_ad.m4
11813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11814 // val ^ (-1 ^ (val >>> shift)) ==> eon
11815 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11816                          iRegL src1, iRegL src2,
11817                          immI src3, immL_M1 src4) %{
11818   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11819   ins_cost(1.9 * INSN_COST);
11820   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11821 
11822   ins_encode %{
11823     __ eon(as_Register($dst$$reg),
11824               as_Register($src1$$reg),
11825               as_Register($src2$$reg),
11826               Assembler::LSR,
11827               $src3$$constant & 0x3f);
11828   %}
11829 
11830   ins_pipe(ialu_reg_reg_shift);
11831 %}
11832 
11833 // This pattern is automatically generated from aarch64_ad.m4
11834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11835 // val ^ (-1 ^ (val >> shift)) ==> eonw
11836 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11837                          iRegIorL2I src1, iRegIorL2I src2,
11838                          immI src3, immI_M1 src4) %{
11839   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11840   ins_cost(1.9 * INSN_COST);
11841   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11842 
11843   ins_encode %{
11844     __ eonw(as_Register($dst$$reg),
11845               as_Register($src1$$reg),
11846               as_Register($src2$$reg),
11847               Assembler::ASR,
11848               $src3$$constant & 0x1f);
11849   %}
11850 
11851   ins_pipe(ialu_reg_reg_shift);
11852 %}
11853 
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 // val ^ (-1 ^ (val >> shift)) ==> eon
11857 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11858                          iRegL src1, iRegL src2,
11859                          immI src3, immL_M1 src4) %{
11860   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11861   ins_cost(1.9 * INSN_COST);
11862   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11863 
11864   ins_encode %{
11865     __ eon(as_Register($dst$$reg),
11866               as_Register($src1$$reg),
11867               as_Register($src2$$reg),
11868               Assembler::ASR,
11869               $src3$$constant & 0x3f);
11870   %}
11871 
11872   ins_pipe(ialu_reg_reg_shift);
11873 %}
11874 
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 // val ^ (-1 ^ (val ror shift)) ==> eonw
11878 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11879                          iRegIorL2I src1, iRegIorL2I src2,
11880                          immI src3, immI_M1 src4) %{
11881   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11882   ins_cost(1.9 * INSN_COST);
11883   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11884 
11885   ins_encode %{
11886     __ eonw(as_Register($dst$$reg),
11887               as_Register($src1$$reg),
11888               as_Register($src2$$reg),
11889               Assembler::ROR,
11890               $src3$$constant & 0x1f);
11891   %}
11892 
11893   ins_pipe(ialu_reg_reg_shift);
11894 %}
11895 
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 // val ^ (-1 ^ (val ror shift)) ==> eon
11899 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11900                          iRegL src1, iRegL src2,
11901                          immI src3, immL_M1 src4) %{
11902   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11903   ins_cost(1.9 * INSN_COST);
11904   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11905 
11906   ins_encode %{
11907     __ eon(as_Register($dst$$reg),
11908               as_Register($src1$$reg),
11909               as_Register($src2$$reg),
11910               Assembler::ROR,
11911               $src3$$constant & 0x3f);
11912   %}
11913 
11914   ins_pipe(ialu_reg_reg_shift);
11915 %}
11916 
11917 // This pattern is automatically generated from aarch64_ad.m4
11918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11919 // val ^ (-1 ^ (val << shift)) ==> eonw
11920 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11921                          iRegIorL2I src1, iRegIorL2I src2,
11922                          immI src3, immI_M1 src4) %{
11923   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11924   ins_cost(1.9 * INSN_COST);
11925   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11926 
11927   ins_encode %{
11928     __ eonw(as_Register($dst$$reg),
11929               as_Register($src1$$reg),
11930               as_Register($src2$$reg),
11931               Assembler::LSL,
11932               $src3$$constant & 0x1f);
11933   %}
11934 
11935   ins_pipe(ialu_reg_reg_shift);
11936 %}
11937 
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 // val ^ (-1 ^ (val << shift)) ==> eon
11941 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11942                          iRegL src1, iRegL src2,
11943                          immI src3, immL_M1 src4) %{
11944   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11945   ins_cost(1.9 * INSN_COST);
11946   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11947 
11948   ins_encode %{
11949     __ eon(as_Register($dst$$reg),
11950               as_Register($src1$$reg),
11951               as_Register($src2$$reg),
11952               Assembler::LSL,
11953               $src3$$constant & 0x3f);
11954   %}
11955 
11956   ins_pipe(ialu_reg_reg_shift);
11957 %}
11958 
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 // val | (-1 ^ (val >>> shift)) ==> ornw
11962 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11963                          iRegIorL2I src1, iRegIorL2I src2,
11964                          immI src3, immI_M1 src4) %{
11965   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11966   ins_cost(1.9 * INSN_COST);
11967   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11968 
11969   ins_encode %{
11970     __ ornw(as_Register($dst$$reg),
11971               as_Register($src1$$reg),
11972               as_Register($src2$$reg),
11973               Assembler::LSR,
11974               $src3$$constant & 0x1f);
11975   %}
11976 
11977   ins_pipe(ialu_reg_reg_shift);
11978 %}
11979 
11980 // This pattern is automatically generated from aarch64_ad.m4
11981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11982 // val | (-1 ^ (val >>> shift)) ==> orn
11983 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11984                          iRegL src1, iRegL src2,
11985                          immI src3, immL_M1 src4) %{
11986   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11987   ins_cost(1.9 * INSN_COST);
11988   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11989 
11990   ins_encode %{
11991     __ orn(as_Register($dst$$reg),
11992               as_Register($src1$$reg),
11993               as_Register($src2$$reg),
11994               Assembler::LSR,
11995               $src3$$constant & 0x3f);
11996   %}
11997 
11998   ins_pipe(ialu_reg_reg_shift);
11999 %}
12000 
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 // val | (-1 ^ (val >> shift)) ==> ornw
12004 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12005                          iRegIorL2I src1, iRegIorL2I src2,
12006                          immI src3, immI_M1 src4) %{
12007   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12008   ins_cost(1.9 * INSN_COST);
12009   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12010 
12011   ins_encode %{
12012     __ ornw(as_Register($dst$$reg),
12013               as_Register($src1$$reg),
12014               as_Register($src2$$reg),
12015               Assembler::ASR,
12016               $src3$$constant & 0x1f);
12017   %}
12018 
12019   ins_pipe(ialu_reg_reg_shift);
12020 %}
12021 
12022 // This pattern is automatically generated from aarch64_ad.m4
12023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12024 // val | (-1 ^ (val >> shift)) ==> orn
12025 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12026                          iRegL src1, iRegL src2,
12027                          immI src3, immL_M1 src4) %{
12028   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12029   ins_cost(1.9 * INSN_COST);
12030   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12031 
12032   ins_encode %{
12033     __ orn(as_Register($dst$$reg),
12034               as_Register($src1$$reg),
12035               as_Register($src2$$reg),
12036               Assembler::ASR,
12037               $src3$$constant & 0x3f);
12038   %}
12039 
12040   ins_pipe(ialu_reg_reg_shift);
12041 %}
12042 
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 // val | (-1 ^ (val ror shift)) ==> ornw
12046 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12047                          iRegIorL2I src1, iRegIorL2I src2,
12048                          immI src3, immI_M1 src4) %{
12049   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12050   ins_cost(1.9 * INSN_COST);
12051   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12052 
12053   ins_encode %{
12054     __ ornw(as_Register($dst$$reg),
12055               as_Register($src1$$reg),
12056               as_Register($src2$$reg),
12057               Assembler::ROR,
12058               $src3$$constant & 0x1f);
12059   %}
12060 
12061   ins_pipe(ialu_reg_reg_shift);
12062 %}
12063 
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 // val | (-1 ^ (val ror shift)) ==> orn
12067 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12068                          iRegL src1, iRegL src2,
12069                          immI src3, immL_M1 src4) %{
12070   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12071   ins_cost(1.9 * INSN_COST);
12072   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12073 
12074   ins_encode %{
12075     __ orn(as_Register($dst$$reg),
12076               as_Register($src1$$reg),
12077               as_Register($src2$$reg),
12078               Assembler::ROR,
12079               $src3$$constant & 0x3f);
12080   %}
12081 
12082   ins_pipe(ialu_reg_reg_shift);
12083 %}
12084 
12085 // This pattern is automatically generated from aarch64_ad.m4
12086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12087 // val | (-1 ^ (val << shift)) ==> ornw
12088 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12089                          iRegIorL2I src1, iRegIorL2I src2,
12090                          immI src3, immI_M1 src4) %{
12091   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12092   ins_cost(1.9 * INSN_COST);
12093   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12094 
12095   ins_encode %{
12096     __ ornw(as_Register($dst$$reg),
12097               as_Register($src1$$reg),
12098               as_Register($src2$$reg),
12099               Assembler::LSL,
12100               $src3$$constant & 0x1f);
12101   %}
12102 
12103   ins_pipe(ialu_reg_reg_shift);
12104 %}
12105 
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 // val | (-1 ^ (val << shift)) ==> orn
12109 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12110                          iRegL src1, iRegL src2,
12111                          immI src3, immL_M1 src4) %{
12112   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12113   ins_cost(1.9 * INSN_COST);
12114   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12115 
12116   ins_encode %{
12117     __ orn(as_Register($dst$$reg),
12118               as_Register($src1$$reg),
12119               as_Register($src2$$reg),
12120               Assembler::LSL,
12121               $src3$$constant & 0x3f);
12122   %}
12123 
12124   ins_pipe(ialu_reg_reg_shift);
12125 %}
12126 
12127 // This pattern is automatically generated from aarch64_ad.m4
12128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12129 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12130                          iRegIorL2I src1, iRegIorL2I src2,
12131                          immI src3) %{
12132   match(Set dst (AndI src1 (URShiftI src2 src3)));
12133 
12134   ins_cost(1.9 * INSN_COST);
12135   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12136 
12137   ins_encode %{
12138     __ andw(as_Register($dst$$reg),
12139               as_Register($src1$$reg),
12140               as_Register($src2$$reg),
12141               Assembler::LSR,
12142               $src3$$constant & 0x1f);
12143   %}
12144 
12145   ins_pipe(ialu_reg_reg_shift);
12146 %}
12147 
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12151                          iRegL src1, iRegL src2,
12152                          immI src3) %{
12153   match(Set dst (AndL src1 (URShiftL src2 src3)));
12154 
12155   ins_cost(1.9 * INSN_COST);
12156   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12157 
12158   ins_encode %{
12159     __ andr(as_Register($dst$$reg),
12160               as_Register($src1$$reg),
12161               as_Register($src2$$reg),
12162               Assembler::LSR,
12163               $src3$$constant & 0x3f);
12164   %}
12165 
12166   ins_pipe(ialu_reg_reg_shift);
12167 %}
12168 
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12172                          iRegIorL2I src1, iRegIorL2I src2,
12173                          immI src3) %{
12174   match(Set dst (AndI src1 (RShiftI src2 src3)));
12175 
12176   ins_cost(1.9 * INSN_COST);
12177   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12178 
12179   ins_encode %{
12180     __ andw(as_Register($dst$$reg),
12181               as_Register($src1$$reg),
12182               as_Register($src2$$reg),
12183               Assembler::ASR,
12184               $src3$$constant & 0x1f);
12185   %}
12186 
12187   ins_pipe(ialu_reg_reg_shift);
12188 %}
12189 
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12193                          iRegL src1, iRegL src2,
12194                          immI src3) %{
12195   match(Set dst (AndL src1 (RShiftL src2 src3)));
12196 
12197   ins_cost(1.9 * INSN_COST);
12198   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12199 
12200   ins_encode %{
12201     __ andr(as_Register($dst$$reg),
12202               as_Register($src1$$reg),
12203               as_Register($src2$$reg),
12204               Assembler::ASR,
12205               $src3$$constant & 0x3f);
12206   %}
12207 
12208   ins_pipe(ialu_reg_reg_shift);
12209 %}
12210 
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12214                          iRegIorL2I src1, iRegIorL2I src2,
12215                          immI src3) %{
12216   match(Set dst (AndI src1 (LShiftI src2 src3)));
12217 
12218   ins_cost(1.9 * INSN_COST);
12219   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12220 
12221   ins_encode %{
12222     __ andw(as_Register($dst$$reg),
12223               as_Register($src1$$reg),
12224               as_Register($src2$$reg),
12225               Assembler::LSL,
12226               $src3$$constant & 0x1f);
12227   %}
12228 
12229   ins_pipe(ialu_reg_reg_shift);
12230 %}
12231 
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12235                          iRegL src1, iRegL src2,
12236                          immI src3) %{
12237   match(Set dst (AndL src1 (LShiftL src2 src3)));
12238 
12239   ins_cost(1.9 * INSN_COST);
12240   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12241 
12242   ins_encode %{
12243     __ andr(as_Register($dst$$reg),
12244               as_Register($src1$$reg),
12245               as_Register($src2$$reg),
12246               Assembler::LSL,
12247               $src3$$constant & 0x3f);
12248   %}
12249 
12250   ins_pipe(ialu_reg_reg_shift);
12251 %}
12252 
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12256                          iRegIorL2I src1, iRegIorL2I src2,
12257                          immI src3) %{
12258   match(Set dst (AndI src1 (RotateRight src2 src3)));
12259 
12260   ins_cost(1.9 * INSN_COST);
12261   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12262 
12263   ins_encode %{
12264     __ andw(as_Register($dst$$reg),
12265               as_Register($src1$$reg),
12266               as_Register($src2$$reg),
12267               Assembler::ROR,
12268               $src3$$constant & 0x1f);
12269   %}
12270 
12271   ins_pipe(ialu_reg_reg_shift);
12272 %}
12273 
12274 // This pattern is automatically generated from aarch64_ad.m4
12275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12276 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12277                          iRegL src1, iRegL src2,
12278                          immI src3) %{
12279   match(Set dst (AndL src1 (RotateRight src2 src3)));
12280 
12281   ins_cost(1.9 * INSN_COST);
12282   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12283 
12284   ins_encode %{
12285     __ andr(as_Register($dst$$reg),
12286               as_Register($src1$$reg),
12287               as_Register($src2$$reg),
12288               Assembler::ROR,
12289               $src3$$constant & 0x3f);
12290   %}
12291 
12292   ins_pipe(ialu_reg_reg_shift);
12293 %}
12294 
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12298                          iRegIorL2I src1, iRegIorL2I src2,
12299                          immI src3) %{
12300   match(Set dst (XorI src1 (URShiftI src2 src3)));
12301 
12302   ins_cost(1.9 * INSN_COST);
12303   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12304 
12305   ins_encode %{
12306     __ eorw(as_Register($dst$$reg),
12307               as_Register($src1$$reg),
12308               as_Register($src2$$reg),
12309               Assembler::LSR,
12310               $src3$$constant & 0x1f);
12311   %}
12312 
12313   ins_pipe(ialu_reg_reg_shift);
12314 %}
12315 
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12319                          iRegL src1, iRegL src2,
12320                          immI src3) %{
12321   match(Set dst (XorL src1 (URShiftL src2 src3)));
12322 
12323   ins_cost(1.9 * INSN_COST);
12324   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12325 
12326   ins_encode %{
12327     __ eor(as_Register($dst$$reg),
12328               as_Register($src1$$reg),
12329               as_Register($src2$$reg),
12330               Assembler::LSR,
12331               $src3$$constant & 0x3f);
12332   %}
12333 
12334   ins_pipe(ialu_reg_reg_shift);
12335 %}
12336 
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12340                          iRegIorL2I src1, iRegIorL2I src2,
12341                          immI src3) %{
12342   match(Set dst (XorI src1 (RShiftI src2 src3)));
12343 
12344   ins_cost(1.9 * INSN_COST);
12345   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12346 
12347   ins_encode %{
12348     __ eorw(as_Register($dst$$reg),
12349               as_Register($src1$$reg),
12350               as_Register($src2$$reg),
12351               Assembler::ASR,
12352               $src3$$constant & 0x1f);
12353   %}
12354 
12355   ins_pipe(ialu_reg_reg_shift);
12356 %}
12357 
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12361                          iRegL src1, iRegL src2,
12362                          immI src3) %{
12363   match(Set dst (XorL src1 (RShiftL src2 src3)));
12364 
12365   ins_cost(1.9 * INSN_COST);
12366   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12367 
12368   ins_encode %{
12369     __ eor(as_Register($dst$$reg),
12370               as_Register($src1$$reg),
12371               as_Register($src2$$reg),
12372               Assembler::ASR,
12373               $src3$$constant & 0x3f);
12374   %}
12375 
12376   ins_pipe(ialu_reg_reg_shift);
12377 %}
12378 
12379 // This pattern is automatically generated from aarch64_ad.m4
12380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12381 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12382                          iRegIorL2I src1, iRegIorL2I src2,
12383                          immI src3) %{
12384   match(Set dst (XorI src1 (LShiftI src2 src3)));
12385 
12386   ins_cost(1.9 * INSN_COST);
12387   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12388 
12389   ins_encode %{
12390     __ eorw(as_Register($dst$$reg),
12391               as_Register($src1$$reg),
12392               as_Register($src2$$reg),
12393               Assembler::LSL,
12394               $src3$$constant & 0x1f);
12395   %}
12396 
12397   ins_pipe(ialu_reg_reg_shift);
12398 %}
12399 
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12403                          iRegL src1, iRegL src2,
12404                          immI src3) %{
12405   match(Set dst (XorL src1 (LShiftL src2 src3)));
12406 
12407   ins_cost(1.9 * INSN_COST);
12408   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12409 
12410   ins_encode %{
12411     __ eor(as_Register($dst$$reg),
12412               as_Register($src1$$reg),
12413               as_Register($src2$$reg),
12414               Assembler::LSL,
12415               $src3$$constant & 0x3f);
12416   %}
12417 
12418   ins_pipe(ialu_reg_reg_shift);
12419 %}
12420 
12421 // This pattern is automatically generated from aarch64_ad.m4
12422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12423 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12424                          iRegIorL2I src1, iRegIorL2I src2,
12425                          immI src3) %{
12426   match(Set dst (XorI src1 (RotateRight src2 src3)));
12427 
12428   ins_cost(1.9 * INSN_COST);
12429   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12430 
12431   ins_encode %{
12432     __ eorw(as_Register($dst$$reg),
12433               as_Register($src1$$reg),
12434               as_Register($src2$$reg),
12435               Assembler::ROR,
12436               $src3$$constant & 0x1f);
12437   %}
12438 
12439   ins_pipe(ialu_reg_reg_shift);
12440 %}
12441 
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12445                          iRegL src1, iRegL src2,
12446                          immI src3) %{
12447   match(Set dst (XorL src1 (RotateRight src2 src3)));
12448 
12449   ins_cost(1.9 * INSN_COST);
12450   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12451 
12452   ins_encode %{
12453     __ eor(as_Register($dst$$reg),
12454               as_Register($src1$$reg),
12455               as_Register($src2$$reg),
12456               Assembler::ROR,
12457               $src3$$constant & 0x3f);
12458   %}
12459 
12460   ins_pipe(ialu_reg_reg_shift);
12461 %}
12462 
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12466                          iRegIorL2I src1, iRegIorL2I src2,
12467                          immI src3) %{
12468   match(Set dst (OrI src1 (URShiftI src2 src3)));
12469 
12470   ins_cost(1.9 * INSN_COST);
12471   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12472 
12473   ins_encode %{
12474     __ orrw(as_Register($dst$$reg),
12475               as_Register($src1$$reg),
12476               as_Register($src2$$reg),
12477               Assembler::LSR,
12478               $src3$$constant & 0x1f);
12479   %}
12480 
12481   ins_pipe(ialu_reg_reg_shift);
12482 %}
12483 
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12487                          iRegL src1, iRegL src2,
12488                          immI src3) %{
12489   match(Set dst (OrL src1 (URShiftL src2 src3)));
12490 
12491   ins_cost(1.9 * INSN_COST);
12492   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12493 
12494   ins_encode %{
12495     __ orr(as_Register($dst$$reg),
12496               as_Register($src1$$reg),
12497               as_Register($src2$$reg),
12498               Assembler::LSR,
12499               $src3$$constant & 0x3f);
12500   %}
12501 
12502   ins_pipe(ialu_reg_reg_shift);
12503 %}
12504 
12505 // This pattern is automatically generated from aarch64_ad.m4
12506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12507 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12508                          iRegIorL2I src1, iRegIorL2I src2,
12509                          immI src3) %{
12510   match(Set dst (OrI src1 (RShiftI src2 src3)));
12511 
12512   ins_cost(1.9 * INSN_COST);
12513   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12514 
12515   ins_encode %{
12516     __ orrw(as_Register($dst$$reg),
12517               as_Register($src1$$reg),
12518               as_Register($src2$$reg),
12519               Assembler::ASR,
12520               $src3$$constant & 0x1f);
12521   %}
12522 
12523   ins_pipe(ialu_reg_reg_shift);
12524 %}
12525 
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12529                          iRegL src1, iRegL src2,
12530                          immI src3) %{
12531   match(Set dst (OrL src1 (RShiftL src2 src3)));
12532 
12533   ins_cost(1.9 * INSN_COST);
12534   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12535 
12536   ins_encode %{
12537     __ orr(as_Register($dst$$reg),
12538               as_Register($src1$$reg),
12539               as_Register($src2$$reg),
12540               Assembler::ASR,
12541               $src3$$constant & 0x3f);
12542   %}
12543 
12544   ins_pipe(ialu_reg_reg_shift);
12545 %}
12546 
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12550                          iRegIorL2I src1, iRegIorL2I src2,
12551                          immI src3) %{
12552   match(Set dst (OrI src1 (LShiftI src2 src3)));
12553 
12554   ins_cost(1.9 * INSN_COST);
12555   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12556 
12557   ins_encode %{
12558     __ orrw(as_Register($dst$$reg),
12559               as_Register($src1$$reg),
12560               as_Register($src2$$reg),
12561               Assembler::LSL,
12562               $src3$$constant & 0x1f);
12563   %}
12564 
12565   ins_pipe(ialu_reg_reg_shift);
12566 %}
12567 
12568 // This pattern is automatically generated from aarch64_ad.m4
12569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12570 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12571                          iRegL src1, iRegL src2,
12572                          immI src3) %{
12573   match(Set dst (OrL src1 (LShiftL src2 src3)));
12574 
12575   ins_cost(1.9 * INSN_COST);
12576   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12577 
12578   ins_encode %{
12579     __ orr(as_Register($dst$$reg),
12580               as_Register($src1$$reg),
12581               as_Register($src2$$reg),
12582               Assembler::LSL,
12583               $src3$$constant & 0x3f);
12584   %}
12585 
12586   ins_pipe(ialu_reg_reg_shift);
12587 %}
12588 
12589 // This pattern is automatically generated from aarch64_ad.m4
12590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12591 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12592                          iRegIorL2I src1, iRegIorL2I src2,
12593                          immI src3) %{
12594   match(Set dst (OrI src1 (RotateRight src2 src3)));
12595 
12596   ins_cost(1.9 * INSN_COST);
12597   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12598 
12599   ins_encode %{
12600     __ orrw(as_Register($dst$$reg),
12601               as_Register($src1$$reg),
12602               as_Register($src2$$reg),
12603               Assembler::ROR,
12604               $src3$$constant & 0x1f);
12605   %}
12606 
12607   ins_pipe(ialu_reg_reg_shift);
12608 %}
12609 
12610 // This pattern is automatically generated from aarch64_ad.m4
12611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12612 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12613                          iRegL src1, iRegL src2,
12614                          immI src3) %{
12615   match(Set dst (OrL src1 (RotateRight src2 src3)));
12616 
12617   ins_cost(1.9 * INSN_COST);
12618   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12619 
12620   ins_encode %{
12621     __ orr(as_Register($dst$$reg),
12622               as_Register($src1$$reg),
12623               as_Register($src2$$reg),
12624               Assembler::ROR,
12625               $src3$$constant & 0x3f);
12626   %}
12627 
12628   ins_pipe(ialu_reg_reg_shift);
12629 %}
12630 
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12634                          iRegIorL2I src1, iRegIorL2I src2,
12635                          immI src3) %{
12636   match(Set dst (AddI src1 (URShiftI src2 src3)));
12637 
12638   ins_cost(1.9 * INSN_COST);
12639   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12640 
12641   ins_encode %{
12642     __ addw(as_Register($dst$$reg),
12643               as_Register($src1$$reg),
12644               as_Register($src2$$reg),
12645               Assembler::LSR,
12646               $src3$$constant & 0x1f);
12647   %}
12648 
12649   ins_pipe(ialu_reg_reg_shift);
12650 %}
12651 
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12655                          iRegL src1, iRegL src2,
12656                          immI src3) %{
12657   match(Set dst (AddL src1 (URShiftL src2 src3)));
12658 
12659   ins_cost(1.9 * INSN_COST);
12660   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12661 
12662   ins_encode %{
12663     __ add(as_Register($dst$$reg),
12664               as_Register($src1$$reg),
12665               as_Register($src2$$reg),
12666               Assembler::LSR,
12667               $src3$$constant & 0x3f);
12668   %}
12669 
12670   ins_pipe(ialu_reg_reg_shift);
12671 %}
12672 
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12676                          iRegIorL2I src1, iRegIorL2I src2,
12677                          immI src3) %{
12678   match(Set dst (AddI src1 (RShiftI src2 src3)));
12679 
12680   ins_cost(1.9 * INSN_COST);
12681   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12682 
12683   ins_encode %{
12684     __ addw(as_Register($dst$$reg),
12685               as_Register($src1$$reg),
12686               as_Register($src2$$reg),
12687               Assembler::ASR,
12688               $src3$$constant & 0x1f);
12689   %}
12690 
12691   ins_pipe(ialu_reg_reg_shift);
12692 %}
12693 
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12697                          iRegL src1, iRegL src2,
12698                          immI src3) %{
12699   match(Set dst (AddL src1 (RShiftL src2 src3)));
12700 
12701   ins_cost(1.9 * INSN_COST);
12702   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12703 
12704   ins_encode %{
12705     __ add(as_Register($dst$$reg),
12706               as_Register($src1$$reg),
12707               as_Register($src2$$reg),
12708               Assembler::ASR,
12709               $src3$$constant & 0x3f);
12710   %}
12711 
12712   ins_pipe(ialu_reg_reg_shift);
12713 %}
12714 
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12718                          iRegIorL2I src1, iRegIorL2I src2,
12719                          immI src3) %{
12720   match(Set dst (AddI src1 (LShiftI src2 src3)));
12721 
12722   ins_cost(1.9 * INSN_COST);
12723   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12724 
12725   ins_encode %{
12726     __ addw(as_Register($dst$$reg),
12727               as_Register($src1$$reg),
12728               as_Register($src2$$reg),
12729               Assembler::LSL,
12730               $src3$$constant & 0x1f);
12731   %}
12732 
12733   ins_pipe(ialu_reg_reg_shift);
12734 %}
12735 
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12739                          iRegL src1, iRegL src2,
12740                          immI src3) %{
12741   match(Set dst (AddL src1 (LShiftL src2 src3)));
12742 
12743   ins_cost(1.9 * INSN_COST);
12744   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12745 
12746   ins_encode %{
12747     __ add(as_Register($dst$$reg),
12748               as_Register($src1$$reg),
12749               as_Register($src2$$reg),
12750               Assembler::LSL,
12751               $src3$$constant & 0x3f);
12752   %}
12753 
12754   ins_pipe(ialu_reg_reg_shift);
12755 %}
12756 
12757 // This pattern is automatically generated from aarch64_ad.m4
12758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12759 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12760                          iRegIorL2I src1, iRegIorL2I src2,
12761                          immI src3) %{
12762   match(Set dst (SubI src1 (URShiftI src2 src3)));
12763 
12764   ins_cost(1.9 * INSN_COST);
12765   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12766 
12767   ins_encode %{
12768     __ subw(as_Register($dst$$reg),
12769               as_Register($src1$$reg),
12770               as_Register($src2$$reg),
12771               Assembler::LSR,
12772               $src3$$constant & 0x1f);
12773   %}
12774 
12775   ins_pipe(ialu_reg_reg_shift);
12776 %}
12777 
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12781                          iRegL src1, iRegL src2,
12782                          immI src3) %{
12783   match(Set dst (SubL src1 (URShiftL src2 src3)));
12784 
12785   ins_cost(1.9 * INSN_COST);
12786   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12787 
12788   ins_encode %{
12789     __ sub(as_Register($dst$$reg),
12790               as_Register($src1$$reg),
12791               as_Register($src2$$reg),
12792               Assembler::LSR,
12793               $src3$$constant & 0x3f);
12794   %}
12795 
12796   ins_pipe(ialu_reg_reg_shift);
12797 %}
12798 
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12802                          iRegIorL2I src1, iRegIorL2I src2,
12803                          immI src3) %{
12804   match(Set dst (SubI src1 (RShiftI src2 src3)));
12805 
12806   ins_cost(1.9 * INSN_COST);
12807   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12808 
12809   ins_encode %{
12810     __ subw(as_Register($dst$$reg),
12811               as_Register($src1$$reg),
12812               as_Register($src2$$reg),
12813               Assembler::ASR,
12814               $src3$$constant & 0x1f);
12815   %}
12816 
12817   ins_pipe(ialu_reg_reg_shift);
12818 %}
12819 
12820 // This pattern is automatically generated from aarch64_ad.m4
12821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12822 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12823                          iRegL src1, iRegL src2,
12824                          immI src3) %{
12825   match(Set dst (SubL src1 (RShiftL src2 src3)));
12826 
12827   ins_cost(1.9 * INSN_COST);
12828   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12829 
12830   ins_encode %{
12831     __ sub(as_Register($dst$$reg),
12832               as_Register($src1$$reg),
12833               as_Register($src2$$reg),
12834               Assembler::ASR,
12835               $src3$$constant & 0x3f);
12836   %}
12837 
12838   ins_pipe(ialu_reg_reg_shift);
12839 %}
12840 
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12844                          iRegIorL2I src1, iRegIorL2I src2,
12845                          immI src3) %{
12846   match(Set dst (SubI src1 (LShiftI src2 src3)));
12847 
12848   ins_cost(1.9 * INSN_COST);
12849   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12850 
12851   ins_encode %{
12852     __ subw(as_Register($dst$$reg),
12853               as_Register($src1$$reg),
12854               as_Register($src2$$reg),
12855               Assembler::LSL,
12856               $src3$$constant & 0x1f);
12857   %}
12858 
12859   ins_pipe(ialu_reg_reg_shift);
12860 %}
12861 
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12865                          iRegL src1, iRegL src2,
12866                          immI src3) %{
12867   match(Set dst (SubL src1 (LShiftL src2 src3)));
12868 
12869   ins_cost(1.9 * INSN_COST);
12870   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12871 
12872   ins_encode %{
12873     __ sub(as_Register($dst$$reg),
12874               as_Register($src1$$reg),
12875               as_Register($src2$$reg),
12876               Assembler::LSL,
12877               $src3$$constant & 0x3f);
12878   %}
12879 
12880   ins_pipe(ialu_reg_reg_shift);
12881 %}
12882 
12883 // This pattern is automatically generated from aarch64_ad.m4
12884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885 
12886 // Shift Left followed by Shift Right.
12887 // This idiom is used by the compiler for the i2b bytecode etc.
12888 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12889 %{
12890   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12891   ins_cost(INSN_COST * 2);
12892   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12893   ins_encode %{
12894     int lshift = $lshift_count$$constant & 63;
12895     int rshift = $rshift_count$$constant & 63;
12896     int s = 63 - lshift;
12897     int r = (rshift - lshift) & 63;
12898     __ sbfm(as_Register($dst$$reg),
12899             as_Register($src$$reg),
12900             r, s);
12901   %}
12902 
12903   ins_pipe(ialu_reg_shift);
12904 %}
12905 
12906 // This pattern is automatically generated from aarch64_ad.m4
12907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12908 
12909 // Shift Left followed by Shift Right.
12910 // This idiom is used by the compiler for the i2b bytecode etc.
12911 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12912 %{
12913   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12914   ins_cost(INSN_COST * 2);
12915   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12916   ins_encode %{
12917     int lshift = $lshift_count$$constant & 31;
12918     int rshift = $rshift_count$$constant & 31;
12919     int s = 31 - lshift;
12920     int r = (rshift - lshift) & 31;
12921     __ sbfmw(as_Register($dst$$reg),
12922             as_Register($src$$reg),
12923             r, s);
12924   %}
12925 
12926   ins_pipe(ialu_reg_shift);
12927 %}
12928 
12929 // This pattern is automatically generated from aarch64_ad.m4
12930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12931 
12932 // Shift Left followed by Shift Right.
12933 // This idiom is used by the compiler for the i2b bytecode etc.
12934 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12935 %{
12936   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12937   ins_cost(INSN_COST * 2);
12938   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12939   ins_encode %{
12940     int lshift = $lshift_count$$constant & 63;
12941     int rshift = $rshift_count$$constant & 63;
12942     int s = 63 - lshift;
12943     int r = (rshift - lshift) & 63;
12944     __ ubfm(as_Register($dst$$reg),
12945             as_Register($src$$reg),
12946             r, s);
12947   %}
12948 
12949   ins_pipe(ialu_reg_shift);
12950 %}
12951 
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 
12955 // Shift Left followed by Shift Right.
12956 // This idiom is used by the compiler for the i2b bytecode etc.
12957 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12958 %{
12959   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12960   ins_cost(INSN_COST * 2);
12961   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12962   ins_encode %{
12963     int lshift = $lshift_count$$constant & 31;
12964     int rshift = $rshift_count$$constant & 31;
12965     int s = 31 - lshift;
12966     int r = (rshift - lshift) & 31;
12967     __ ubfmw(as_Register($dst$$reg),
12968             as_Register($src$$reg),
12969             r, s);
12970   %}
12971 
12972   ins_pipe(ialu_reg_shift);
12973 %}
12974 
12975 // Bitfield extract with shift & mask
12976 
12977 // This pattern is automatically generated from aarch64_ad.m4
12978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12979 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12980 %{
12981   match(Set dst (AndI (URShiftI src rshift) mask));
12982   // Make sure we are not going to exceed what ubfxw can do.
12983   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12984 
12985   ins_cost(INSN_COST);
12986   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12987   ins_encode %{
12988     int rshift = $rshift$$constant & 31;
12989     intptr_t mask = $mask$$constant;
12990     int width = exact_log2(mask+1);
12991     __ ubfxw(as_Register($dst$$reg),
12992             as_Register($src$$reg), rshift, width);
12993   %}
12994   ins_pipe(ialu_reg_shift);
12995 %}
12996 
12997 // This pattern is automatically generated from aarch64_ad.m4
12998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12999 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13000 %{
13001   match(Set dst (AndL (URShiftL src rshift) mask));
13002   // Make sure we are not going to exceed what ubfx can do.
13003   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13004 
13005   ins_cost(INSN_COST);
13006   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13007   ins_encode %{
13008     int rshift = $rshift$$constant & 63;
13009     intptr_t mask = $mask$$constant;
13010     int width = exact_log2_long(mask+1);
13011     __ ubfx(as_Register($dst$$reg),
13012             as_Register($src$$reg), rshift, width);
13013   %}
13014   ins_pipe(ialu_reg_shift);
13015 %}
13016 
13017 
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 
13021 // We can use ubfx when extending an And with a mask when we know mask
13022 // is positive.  We know that because immI_bitmask guarantees it.
13023 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13024 %{
13025   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13026   // Make sure we are not going to exceed what ubfxw can do.
13027   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13028 
13029   ins_cost(INSN_COST * 2);
13030   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13031   ins_encode %{
13032     int rshift = $rshift$$constant & 31;
13033     intptr_t mask = $mask$$constant;
13034     int width = exact_log2(mask+1);
13035     __ ubfx(as_Register($dst$$reg),
13036             as_Register($src$$reg), rshift, width);
13037   %}
13038   ins_pipe(ialu_reg_shift);
13039 %}
13040 
13041 
13042 // This pattern is automatically generated from aarch64_ad.m4
13043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13044 
13045 // We can use ubfiz when masking by a positive number and then left shifting the result.
13046 // We know that the mask is positive because immI_bitmask guarantees it.
13047 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13048 %{
13049   match(Set dst (LShiftI (AndI src mask) lshift));
13050   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13051 
13052   ins_cost(INSN_COST);
13053   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13054   ins_encode %{
13055     int lshift = $lshift$$constant & 31;
13056     intptr_t mask = $mask$$constant;
13057     int width = exact_log2(mask+1);
13058     __ ubfizw(as_Register($dst$$reg),
13059           as_Register($src$$reg), lshift, width);
13060   %}
13061   ins_pipe(ialu_reg_shift);
13062 %}
13063 
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 
13067 // We can use ubfiz when masking by a positive number and then left shifting the result.
13068 // We know that the mask is positive because immL_bitmask guarantees it.
13069 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13070 %{
13071   match(Set dst (LShiftL (AndL src mask) lshift));
13072   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13073 
13074   ins_cost(INSN_COST);
13075   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13076   ins_encode %{
13077     int lshift = $lshift$$constant & 63;
13078     intptr_t mask = $mask$$constant;
13079     int width = exact_log2_long(mask+1);
13080     __ ubfiz(as_Register($dst$$reg),
13081           as_Register($src$$reg), lshift, width);
13082   %}
13083   ins_pipe(ialu_reg_shift);
13084 %}
13085 
13086 // This pattern is automatically generated from aarch64_ad.m4
13087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13088 
13089 // We can use ubfiz when masking by a positive number and then left shifting the result.
13090 // We know that the mask is positive because immI_bitmask guarantees it.
13091 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13092 %{
13093   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13094   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13095 
13096   ins_cost(INSN_COST);
13097   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13098   ins_encode %{
13099     int lshift = $lshift$$constant & 31;
13100     intptr_t mask = $mask$$constant;
13101     int width = exact_log2(mask+1);
13102     __ ubfizw(as_Register($dst$$reg),
13103           as_Register($src$$reg), lshift, width);
13104   %}
13105   ins_pipe(ialu_reg_shift);
13106 %}
13107 
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 
13111 // We can use ubfiz when masking by a positive number and then left shifting the result.
13112 // We know that the mask is positive because immL_bitmask guarantees it.
13113 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13114 %{
13115   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13116   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13117 
13118   ins_cost(INSN_COST);
13119   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13120   ins_encode %{
13121     int lshift = $lshift$$constant & 63;
13122     intptr_t mask = $mask$$constant;
13123     int width = exact_log2_long(mask+1);
13124     __ ubfiz(as_Register($dst$$reg),
13125           as_Register($src$$reg), lshift, width);
13126   %}
13127   ins_pipe(ialu_reg_shift);
13128 %}
13129 
13130 
13131 // This pattern is automatically generated from aarch64_ad.m4
13132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13133 
13134 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13135 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13136 %{
13137   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13138   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13139 
13140   ins_cost(INSN_COST);
13141   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13142   ins_encode %{
13143     int lshift = $lshift$$constant & 63;
13144     intptr_t mask = $mask$$constant;
13145     int width = exact_log2(mask+1);
13146     __ ubfiz(as_Register($dst$$reg),
13147              as_Register($src$$reg), lshift, width);
13148   %}
13149   ins_pipe(ialu_reg_shift);
13150 %}
13151 
13152 // This pattern is automatically generated from aarch64_ad.m4
13153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13154 
13155 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13156 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13157 %{
13158   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13159   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13160 
13161   ins_cost(INSN_COST);
13162   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13163   ins_encode %{
13164     int lshift = $lshift$$constant & 31;
13165     intptr_t mask = $mask$$constant;
13166     int width = exact_log2(mask+1);
13167     __ ubfiz(as_Register($dst$$reg),
13168              as_Register($src$$reg), lshift, width);
13169   %}
13170   ins_pipe(ialu_reg_shift);
13171 %}
13172 
13173 // This pattern is automatically generated from aarch64_ad.m4
13174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13175 
13176 // Can skip int2long conversions after AND with small bitmask
13177 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13178 %{
13179   match(Set dst (ConvI2L (AndI src msk)));
13180   ins_cost(INSN_COST);
13181   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13182   ins_encode %{
13183     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13184   %}
13185   ins_pipe(ialu_reg_shift);
13186 %}
13187 
13188 
13189 // Rotations
13190 // This pattern is automatically generated from aarch64_ad.m4
13191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13192 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13193 %{
13194   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13195   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13196 
13197   ins_cost(INSN_COST);
13198   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13199 
13200   ins_encode %{
13201     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13202             $rshift$$constant & 63);
13203   %}
13204   ins_pipe(ialu_reg_reg_extr);
13205 %}
13206 
13207 
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13211 %{
13212   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13213   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13214 
13215   ins_cost(INSN_COST);
13216   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13217 
13218   ins_encode %{
13219     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13220             $rshift$$constant & 31);
13221   %}
13222   ins_pipe(ialu_reg_reg_extr);
13223 %}
13224 
13225 
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13229 %{
13230   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13231   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13232 
13233   ins_cost(INSN_COST);
13234   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13235 
13236   ins_encode %{
13237     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13238             $rshift$$constant & 63);
13239   %}
13240   ins_pipe(ialu_reg_reg_extr);
13241 %}
13242 
13243 
13244 // This pattern is automatically generated from aarch64_ad.m4
13245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13246 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13247 %{
13248   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13249   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13250 
13251   ins_cost(INSN_COST);
13252   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13253 
13254   ins_encode %{
13255     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13256             $rshift$$constant & 31);
13257   %}
13258   ins_pipe(ialu_reg_reg_extr);
13259 %}
13260 
13261 
13262 // This pattern is automatically generated from aarch64_ad.m4
13263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13264 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13265 %{
13266   match(Set dst (RotateRight src shift));
13267 
13268   ins_cost(INSN_COST);
13269   format %{ "ror    $dst, $src, $shift" %}
13270 
13271   ins_encode %{
13272      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13273                $shift$$constant & 0x1f);
13274   %}
13275   ins_pipe(ialu_reg_reg_vshift);
13276 %}
13277 
13278 // This pattern is automatically generated from aarch64_ad.m4
13279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13280 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13281 %{
13282   match(Set dst (RotateRight src shift));
13283 
13284   ins_cost(INSN_COST);
13285   format %{ "ror    $dst, $src, $shift" %}
13286 
13287   ins_encode %{
13288      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13289                $shift$$constant & 0x3f);
13290   %}
13291   ins_pipe(ialu_reg_reg_vshift);
13292 %}
13293 
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13297 %{
13298   match(Set dst (RotateRight src shift));
13299 
13300   ins_cost(INSN_COST);
13301   format %{ "ror    $dst, $src, $shift" %}
13302 
13303   ins_encode %{
13304      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13305   %}
13306   ins_pipe(ialu_reg_reg_vshift);
13307 %}
13308 
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13312 %{
13313   match(Set dst (RotateRight src shift));
13314 
13315   ins_cost(INSN_COST);
13316   format %{ "ror    $dst, $src, $shift" %}
13317 
13318   ins_encode %{
13319      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13320   %}
13321   ins_pipe(ialu_reg_reg_vshift);
13322 %}
13323 
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13327 %{
13328   match(Set dst (RotateLeft src shift));
13329 
13330   ins_cost(INSN_COST);
13331   format %{ "rol    $dst, $src, $shift" %}
13332 
13333   ins_encode %{
13334      __ subw(rscratch1, zr, as_Register($shift$$reg));
13335      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13336   %}
13337   ins_pipe(ialu_reg_reg_vshift);
13338 %}
13339 
13340 // This pattern is automatically generated from aarch64_ad.m4
13341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13342 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13343 %{
13344   match(Set dst (RotateLeft src shift));
13345 
13346   ins_cost(INSN_COST);
13347   format %{ "rol    $dst, $src, $shift" %}
13348 
13349   ins_encode %{
13350      __ subw(rscratch1, zr, as_Register($shift$$reg));
13351      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13352   %}
13353   ins_pipe(ialu_reg_reg_vshift);
13354 %}
13355 
13356 
13357 // Add/subtract (extended)
13358 
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13362 %{
13363   match(Set dst (AddL src1 (ConvI2L src2)));
13364   ins_cost(INSN_COST);
13365   format %{ "add  $dst, $src1, $src2, sxtw" %}
13366 
13367    ins_encode %{
13368      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13369             as_Register($src2$$reg), ext::sxtw);
13370    %}
13371   ins_pipe(ialu_reg_reg);
13372 %}
13373 
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13377 %{
13378   match(Set dst (SubL src1 (ConvI2L src2)));
13379   ins_cost(INSN_COST);
13380   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13381 
13382    ins_encode %{
13383      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13384             as_Register($src2$$reg), ext::sxtw);
13385    %}
13386   ins_pipe(ialu_reg_reg);
13387 %}
13388 
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13392 %{
13393   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13394   ins_cost(INSN_COST);
13395   format %{ "add  $dst, $src1, $src2, sxth" %}
13396 
13397    ins_encode %{
13398      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13399             as_Register($src2$$reg), ext::sxth);
13400    %}
13401   ins_pipe(ialu_reg_reg);
13402 %}
13403 
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13407 %{
13408   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13409   ins_cost(INSN_COST);
13410   format %{ "add  $dst, $src1, $src2, sxtb" %}
13411 
13412    ins_encode %{
13413      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13414             as_Register($src2$$reg), ext::sxtb);
13415    %}
13416   ins_pipe(ialu_reg_reg);
13417 %}
13418 
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13422 %{
13423   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13424   ins_cost(INSN_COST);
13425   format %{ "add  $dst, $src1, $src2, uxtb" %}
13426 
13427    ins_encode %{
13428      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13429             as_Register($src2$$reg), ext::uxtb);
13430    %}
13431   ins_pipe(ialu_reg_reg);
13432 %}
13433 
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13437 %{
13438   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13439   ins_cost(INSN_COST);
13440   format %{ "add  $dst, $src1, $src2, sxth" %}
13441 
13442    ins_encode %{
13443      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13444             as_Register($src2$$reg), ext::sxth);
13445    %}
13446   ins_pipe(ialu_reg_reg);
13447 %}
13448 
13449 // This pattern is automatically generated from aarch64_ad.m4
13450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13451 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13452 %{
13453   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13454   ins_cost(INSN_COST);
13455   format %{ "add  $dst, $src1, $src2, sxtw" %}
13456 
13457    ins_encode %{
13458      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13459             as_Register($src2$$reg), ext::sxtw);
13460    %}
13461   ins_pipe(ialu_reg_reg);
13462 %}
13463 
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13467 %{
13468   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13469   ins_cost(INSN_COST);
13470   format %{ "add  $dst, $src1, $src2, sxtb" %}
13471 
13472    ins_encode %{
13473      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13474             as_Register($src2$$reg), ext::sxtb);
13475    %}
13476   ins_pipe(ialu_reg_reg);
13477 %}
13478 
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13482 %{
13483   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13484   ins_cost(INSN_COST);
13485   format %{ "add  $dst, $src1, $src2, uxtb" %}
13486 
13487    ins_encode %{
13488      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13489             as_Register($src2$$reg), ext::uxtb);
13490    %}
13491   ins_pipe(ialu_reg_reg);
13492 %}
13493 
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13497 %{
13498   match(Set dst (AddI src1 (AndI src2 mask)));
13499   ins_cost(INSN_COST);
13500   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13501 
13502    ins_encode %{
13503      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13504             as_Register($src2$$reg), ext::uxtb);
13505    %}
13506   ins_pipe(ialu_reg_reg);
13507 %}
13508 
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13512 %{
13513   match(Set dst (AddI src1 (AndI src2 mask)));
13514   ins_cost(INSN_COST);
13515   format %{ "addw  $dst, $src1, $src2, uxth" %}
13516 
13517    ins_encode %{
13518      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13519             as_Register($src2$$reg), ext::uxth);
13520    %}
13521   ins_pipe(ialu_reg_reg);
13522 %}
13523 
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13527 %{
13528   match(Set dst (AddL src1 (AndL src2 mask)));
13529   ins_cost(INSN_COST);
13530   format %{ "add  $dst, $src1, $src2, uxtb" %}
13531 
13532    ins_encode %{
13533      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13534             as_Register($src2$$reg), ext::uxtb);
13535    %}
13536   ins_pipe(ialu_reg_reg);
13537 %}
13538 
13539 // This pattern is automatically generated from aarch64_ad.m4
13540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13541 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13542 %{
13543   match(Set dst (AddL src1 (AndL src2 mask)));
13544   ins_cost(INSN_COST);
13545   format %{ "add  $dst, $src1, $src2, uxth" %}
13546 
13547    ins_encode %{
13548      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13549             as_Register($src2$$reg), ext::uxth);
13550    %}
13551   ins_pipe(ialu_reg_reg);
13552 %}
13553 
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13557 %{
13558   match(Set dst (AddL src1 (AndL src2 mask)));
13559   ins_cost(INSN_COST);
13560   format %{ "add  $dst, $src1, $src2, uxtw" %}
13561 
13562    ins_encode %{
13563      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13564             as_Register($src2$$reg), ext::uxtw);
13565    %}
13566   ins_pipe(ialu_reg_reg);
13567 %}
13568 
13569 // This pattern is automatically generated from aarch64_ad.m4
13570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13571 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13572 %{
13573   match(Set dst (SubI src1 (AndI src2 mask)));
13574   ins_cost(INSN_COST);
13575   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13576 
13577    ins_encode %{
13578      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13579             as_Register($src2$$reg), ext::uxtb);
13580    %}
13581   ins_pipe(ialu_reg_reg);
13582 %}
13583 
13584 // This pattern is automatically generated from aarch64_ad.m4
13585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13586 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13587 %{
13588   match(Set dst (SubI src1 (AndI src2 mask)));
13589   ins_cost(INSN_COST);
13590   format %{ "subw  $dst, $src1, $src2, uxth" %}
13591 
13592    ins_encode %{
13593      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13594             as_Register($src2$$reg), ext::uxth);
13595    %}
13596   ins_pipe(ialu_reg_reg);
13597 %}
13598 
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13602 %{
13603   match(Set dst (SubL src1 (AndL src2 mask)));
13604   ins_cost(INSN_COST);
13605   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13606 
13607    ins_encode %{
13608      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13609             as_Register($src2$$reg), ext::uxtb);
13610    %}
13611   ins_pipe(ialu_reg_reg);
13612 %}
13613 
13614 // This pattern is automatically generated from aarch64_ad.m4
13615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13616 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13617 %{
13618   match(Set dst (SubL src1 (AndL src2 mask)));
13619   ins_cost(INSN_COST);
13620   format %{ "sub  $dst, $src1, $src2, uxth" %}
13621 
13622    ins_encode %{
13623      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13624             as_Register($src2$$reg), ext::uxth);
13625    %}
13626   ins_pipe(ialu_reg_reg);
13627 %}
13628 
13629 // This pattern is automatically generated from aarch64_ad.m4
13630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13631 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13632 %{
13633   match(Set dst (SubL src1 (AndL src2 mask)));
13634   ins_cost(INSN_COST);
13635   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13636 
13637    ins_encode %{
13638      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13639             as_Register($src2$$reg), ext::uxtw);
13640    %}
13641   ins_pipe(ialu_reg_reg);
13642 %}
13643 
13644 
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13648 %{
13649   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13650   ins_cost(1.9 * INSN_COST);
13651   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13652 
13653    ins_encode %{
13654      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13655             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13656    %}
13657   ins_pipe(ialu_reg_reg_shift);
13658 %}
13659 
13660 // This pattern is automatically generated from aarch64_ad.m4
13661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13662 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13663 %{
13664   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13665   ins_cost(1.9 * INSN_COST);
13666   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13667 
13668    ins_encode %{
13669      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13670             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13671    %}
13672   ins_pipe(ialu_reg_reg_shift);
13673 %}
13674 
13675 // This pattern is automatically generated from aarch64_ad.m4
13676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13677 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13678 %{
13679   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13680   ins_cost(1.9 * INSN_COST);
13681   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13682 
13683    ins_encode %{
13684      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13685             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13686    %}
13687   ins_pipe(ialu_reg_reg_shift);
13688 %}
13689 
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13693 %{
13694   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13695   ins_cost(1.9 * INSN_COST);
13696   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13697 
13698    ins_encode %{
13699      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13700             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13701    %}
13702   ins_pipe(ialu_reg_reg_shift);
13703 %}
13704 
13705 // This pattern is automatically generated from aarch64_ad.m4
13706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13707 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13708 %{
13709   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13710   ins_cost(1.9 * INSN_COST);
13711   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13712 
13713    ins_encode %{
13714      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13715             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13716    %}
13717   ins_pipe(ialu_reg_reg_shift);
13718 %}
13719 
13720 // This pattern is automatically generated from aarch64_ad.m4
13721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13722 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13723 %{
13724   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13725   ins_cost(1.9 * INSN_COST);
13726   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13727 
13728    ins_encode %{
13729      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13730             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13731    %}
13732   ins_pipe(ialu_reg_reg_shift);
13733 %}
13734 
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13738 %{
13739   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13740   ins_cost(1.9 * INSN_COST);
13741   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13742 
13743    ins_encode %{
13744      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13745             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13746    %}
13747   ins_pipe(ialu_reg_reg_shift);
13748 %}
13749 
13750 // This pattern is automatically generated from aarch64_ad.m4
13751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13752 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13753 %{
13754   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13755   ins_cost(1.9 * INSN_COST);
13756   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13757 
13758    ins_encode %{
13759      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13760             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13761    %}
13762   ins_pipe(ialu_reg_reg_shift);
13763 %}
13764 
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13768 %{
13769   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13770   ins_cost(1.9 * INSN_COST);
13771   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13772 
13773    ins_encode %{
13774      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13775             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13776    %}
13777   ins_pipe(ialu_reg_reg_shift);
13778 %}
13779 
13780 // This pattern is automatically generated from aarch64_ad.m4
13781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13782 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13783 %{
13784   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13785   ins_cost(1.9 * INSN_COST);
13786   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13787 
13788    ins_encode %{
13789      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13790             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13791    %}
13792   ins_pipe(ialu_reg_reg_shift);
13793 %}
13794 
13795 // This pattern is automatically generated from aarch64_ad.m4
13796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13797 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13798 %{
13799   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13800   ins_cost(1.9 * INSN_COST);
13801   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13802 
13803    ins_encode %{
13804      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13805             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13806    %}
13807   ins_pipe(ialu_reg_reg_shift);
13808 %}
13809 
13810 // This pattern is automatically generated from aarch64_ad.m4
13811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13812 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13813 %{
13814   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13815   ins_cost(1.9 * INSN_COST);
13816   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13817 
13818    ins_encode %{
13819      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13820             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13821    %}
13822   ins_pipe(ialu_reg_reg_shift);
13823 %}
13824 
13825 // This pattern is automatically generated from aarch64_ad.m4
13826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13827 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13828 %{
13829   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13830   ins_cost(1.9 * INSN_COST);
13831   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13832 
13833    ins_encode %{
13834      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13835             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13836    %}
13837   ins_pipe(ialu_reg_reg_shift);
13838 %}
13839 
13840 // This pattern is automatically generated from aarch64_ad.m4
13841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13842 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13843 %{
13844   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13845   ins_cost(1.9 * INSN_COST);
13846   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13847 
13848    ins_encode %{
13849      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13850             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13851    %}
13852   ins_pipe(ialu_reg_reg_shift);
13853 %}
13854 
13855 // This pattern is automatically generated from aarch64_ad.m4
13856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13857 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13858 %{
13859   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13860   ins_cost(1.9 * INSN_COST);
13861   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13862 
13863    ins_encode %{
13864      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13865             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13866    %}
13867   ins_pipe(ialu_reg_reg_shift);
13868 %}
13869 
13870 // This pattern is automatically generated from aarch64_ad.m4
13871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13872 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13873 %{
13874   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13875   ins_cost(1.9 * INSN_COST);
13876   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13877 
13878    ins_encode %{
13879      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13880             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13881    %}
13882   ins_pipe(ialu_reg_reg_shift);
13883 %}
13884 
13885 // This pattern is automatically generated from aarch64_ad.m4
13886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13887 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13888 %{
13889   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13890   ins_cost(1.9 * INSN_COST);
13891   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13892 
13893    ins_encode %{
13894      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13895             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13896    %}
13897   ins_pipe(ialu_reg_reg_shift);
13898 %}
13899 
13900 // This pattern is automatically generated from aarch64_ad.m4
13901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13902 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13903 %{
13904   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13905   ins_cost(1.9 * INSN_COST);
13906   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13907 
13908    ins_encode %{
13909      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13910             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13911    %}
13912   ins_pipe(ialu_reg_reg_shift);
13913 %}
13914 
13915 // This pattern is automatically generated from aarch64_ad.m4
13916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13917 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13918 %{
13919   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13920   ins_cost(1.9 * INSN_COST);
13921   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13922 
13923    ins_encode %{
13924      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13925             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13926    %}
13927   ins_pipe(ialu_reg_reg_shift);
13928 %}
13929 
13930 // This pattern is automatically generated from aarch64_ad.m4
13931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13932 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13933 %{
13934   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13935   ins_cost(1.9 * INSN_COST);
13936   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13937 
13938    ins_encode %{
13939      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13940             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13941    %}
13942   ins_pipe(ialu_reg_reg_shift);
13943 %}
13944 
13945 // This pattern is automatically generated from aarch64_ad.m4
13946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13947 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13948 %{
13949   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13950   ins_cost(1.9 * INSN_COST);
13951   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13952 
13953    ins_encode %{
13954      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13955             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13956    %}
13957   ins_pipe(ialu_reg_reg_shift);
13958 %}
13959 
13960 // This pattern is automatically generated from aarch64_ad.m4
13961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13962 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13963 %{
13964   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13965   ins_cost(1.9 * INSN_COST);
13966   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13967 
13968    ins_encode %{
13969      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13970             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13971    %}
13972   ins_pipe(ialu_reg_reg_shift);
13973 %}
13974 
13975 
13976 
13977 // END This section of the file is automatically generated. Do not edit --------------
13978 
13979 
13980 // ============================================================================
13981 // Floating Point Arithmetic Instructions
13982 
13983 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13984   match(Set dst (AddF src1 src2));
13985 
13986   ins_cost(INSN_COST * 5);
13987   format %{ "fadds   $dst, $src1, $src2" %}
13988 
13989   ins_encode %{
13990     __ fadds(as_FloatRegister($dst$$reg),
13991              as_FloatRegister($src1$$reg),
13992              as_FloatRegister($src2$$reg));
13993   %}
13994 
13995   ins_pipe(fp_dop_reg_reg_s);
13996 %}
13997 
13998 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13999   match(Set dst (AddD src1 src2));
14000 
14001   ins_cost(INSN_COST * 5);
14002   format %{ "faddd   $dst, $src1, $src2" %}
14003 
14004   ins_encode %{
14005     __ faddd(as_FloatRegister($dst$$reg),
14006              as_FloatRegister($src1$$reg),
14007              as_FloatRegister($src2$$reg));
14008   %}
14009 
14010   ins_pipe(fp_dop_reg_reg_d);
14011 %}
14012 
14013 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14014   match(Set dst (SubF src1 src2));
14015 
14016   ins_cost(INSN_COST * 5);
14017   format %{ "fsubs   $dst, $src1, $src2" %}
14018 
14019   ins_encode %{
14020     __ fsubs(as_FloatRegister($dst$$reg),
14021              as_FloatRegister($src1$$reg),
14022              as_FloatRegister($src2$$reg));
14023   %}
14024 
14025   ins_pipe(fp_dop_reg_reg_s);
14026 %}
14027 
14028 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14029   match(Set dst (SubD src1 src2));
14030 
14031   ins_cost(INSN_COST * 5);
14032   format %{ "fsubd   $dst, $src1, $src2" %}
14033 
14034   ins_encode %{
14035     __ fsubd(as_FloatRegister($dst$$reg),
14036              as_FloatRegister($src1$$reg),
14037              as_FloatRegister($src2$$reg));
14038   %}
14039 
14040   ins_pipe(fp_dop_reg_reg_d);
14041 %}
14042 
14043 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14044   match(Set dst (MulF src1 src2));
14045 
14046   ins_cost(INSN_COST * 6);
14047   format %{ "fmuls   $dst, $src1, $src2" %}
14048 
14049   ins_encode %{
14050     __ fmuls(as_FloatRegister($dst$$reg),
14051              as_FloatRegister($src1$$reg),
14052              as_FloatRegister($src2$$reg));
14053   %}
14054 
14055   ins_pipe(fp_dop_reg_reg_s);
14056 %}
14057 
14058 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14059   match(Set dst (MulD src1 src2));
14060 
14061   ins_cost(INSN_COST * 6);
14062   format %{ "fmuld   $dst, $src1, $src2" %}
14063 
14064   ins_encode %{
14065     __ fmuld(as_FloatRegister($dst$$reg),
14066              as_FloatRegister($src1$$reg),
14067              as_FloatRegister($src2$$reg));
14068   %}
14069 
14070   ins_pipe(fp_dop_reg_reg_d);
14071 %}
14072 
14073 // src1 * src2 + src3
14074 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14075   predicate(UseFMA);
14076   match(Set dst (FmaF src3 (Binary src1 src2)));
14077 
14078   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14079 
14080   ins_encode %{
14081     __ fmadds(as_FloatRegister($dst$$reg),
14082              as_FloatRegister($src1$$reg),
14083              as_FloatRegister($src2$$reg),
14084              as_FloatRegister($src3$$reg));
14085   %}
14086 
14087   ins_pipe(pipe_class_default);
14088 %}
14089 
14090 // src1 * src2 + src3
14091 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14092   predicate(UseFMA);
14093   match(Set dst (FmaD src3 (Binary src1 src2)));
14094 
14095   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14096 
14097   ins_encode %{
14098     __ fmaddd(as_FloatRegister($dst$$reg),
14099              as_FloatRegister($src1$$reg),
14100              as_FloatRegister($src2$$reg),
14101              as_FloatRegister($src3$$reg));
14102   %}
14103 
14104   ins_pipe(pipe_class_default);
14105 %}
14106 
14107 // -src1 * src2 + src3
14108 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14109   predicate(UseFMA);
14110   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14111   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14112 
14113   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14114 
14115   ins_encode %{
14116     __ fmsubs(as_FloatRegister($dst$$reg),
14117               as_FloatRegister($src1$$reg),
14118               as_FloatRegister($src2$$reg),
14119               as_FloatRegister($src3$$reg));
14120   %}
14121 
14122   ins_pipe(pipe_class_default);
14123 %}
14124 
14125 // -src1 * src2 + src3
14126 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14127   predicate(UseFMA);
14128   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14129   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14130 
14131   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14132 
14133   ins_encode %{
14134     __ fmsubd(as_FloatRegister($dst$$reg),
14135               as_FloatRegister($src1$$reg),
14136               as_FloatRegister($src2$$reg),
14137               as_FloatRegister($src3$$reg));
14138   %}
14139 
14140   ins_pipe(pipe_class_default);
14141 %}
14142 
14143 // -src1 * src2 - src3
14144 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14145   predicate(UseFMA);
14146   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14147   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14148 
14149   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14150 
14151   ins_encode %{
14152     __ fnmadds(as_FloatRegister($dst$$reg),
14153                as_FloatRegister($src1$$reg),
14154                as_FloatRegister($src2$$reg),
14155                as_FloatRegister($src3$$reg));
14156   %}
14157 
14158   ins_pipe(pipe_class_default);
14159 %}
14160 
14161 // -src1 * src2 - src3
14162 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14163   predicate(UseFMA);
14164   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14165   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14166 
14167   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14168 
14169   ins_encode %{
14170     __ fnmaddd(as_FloatRegister($dst$$reg),
14171                as_FloatRegister($src1$$reg),
14172                as_FloatRegister($src2$$reg),
14173                as_FloatRegister($src3$$reg));
14174   %}
14175 
14176   ins_pipe(pipe_class_default);
14177 %}
14178 
14179 // src1 * src2 - src3
14180 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14181   predicate(UseFMA);
14182   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14183 
14184   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14185 
14186   ins_encode %{
14187     __ fnmsubs(as_FloatRegister($dst$$reg),
14188                as_FloatRegister($src1$$reg),
14189                as_FloatRegister($src2$$reg),
14190                as_FloatRegister($src3$$reg));
14191   %}
14192 
14193   ins_pipe(pipe_class_default);
14194 %}
14195 
14196 // src1 * src2 - src3
14197 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14198   predicate(UseFMA);
14199   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14200 
14201   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14202 
14203   ins_encode %{
14204   // n.b. insn name should be fnmsubd
14205     __ fnmsub(as_FloatRegister($dst$$reg),
14206               as_FloatRegister($src1$$reg),
14207               as_FloatRegister($src2$$reg),
14208               as_FloatRegister($src3$$reg));
14209   %}
14210 
14211   ins_pipe(pipe_class_default);
14212 %}
14213 
14214 
14215 // Math.max(FF)F
14216 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14217   match(Set dst (MaxF src1 src2));
14218 
14219   format %{ "fmaxs   $dst, $src1, $src2" %}
14220   ins_encode %{
14221     __ fmaxs(as_FloatRegister($dst$$reg),
14222              as_FloatRegister($src1$$reg),
14223              as_FloatRegister($src2$$reg));
14224   %}
14225 
14226   ins_pipe(fp_dop_reg_reg_s);
14227 %}
14228 
14229 // Math.min(FF)F
14230 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14231   match(Set dst (MinF src1 src2));
14232 
14233   format %{ "fmins   $dst, $src1, $src2" %}
14234   ins_encode %{
14235     __ fmins(as_FloatRegister($dst$$reg),
14236              as_FloatRegister($src1$$reg),
14237              as_FloatRegister($src2$$reg));
14238   %}
14239 
14240   ins_pipe(fp_dop_reg_reg_s);
14241 %}
14242 
14243 // Math.max(DD)D
14244 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14245   match(Set dst (MaxD src1 src2));
14246 
14247   format %{ "fmaxd   $dst, $src1, $src2" %}
14248   ins_encode %{
14249     __ fmaxd(as_FloatRegister($dst$$reg),
14250              as_FloatRegister($src1$$reg),
14251              as_FloatRegister($src2$$reg));
14252   %}
14253 
14254   ins_pipe(fp_dop_reg_reg_d);
14255 %}
14256 
14257 // Math.min(DD)D
14258 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14259   match(Set dst (MinD src1 src2));
14260 
14261   format %{ "fmind   $dst, $src1, $src2" %}
14262   ins_encode %{
14263     __ fmind(as_FloatRegister($dst$$reg),
14264              as_FloatRegister($src1$$reg),
14265              as_FloatRegister($src2$$reg));
14266   %}
14267 
14268   ins_pipe(fp_dop_reg_reg_d);
14269 %}
14270 
14271 
14272 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14273   match(Set dst (DivF src1  src2));
14274 
14275   ins_cost(INSN_COST * 18);
14276   format %{ "fdivs   $dst, $src1, $src2" %}
14277 
14278   ins_encode %{
14279     __ fdivs(as_FloatRegister($dst$$reg),
14280              as_FloatRegister($src1$$reg),
14281              as_FloatRegister($src2$$reg));
14282   %}
14283 
14284   ins_pipe(fp_div_s);
14285 %}
14286 
14287 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14288   match(Set dst (DivD src1  src2));
14289 
14290   ins_cost(INSN_COST * 32);
14291   format %{ "fdivd   $dst, $src1, $src2" %}
14292 
14293   ins_encode %{
14294     __ fdivd(as_FloatRegister($dst$$reg),
14295              as_FloatRegister($src1$$reg),
14296              as_FloatRegister($src2$$reg));
14297   %}
14298 
14299   ins_pipe(fp_div_d);
14300 %}
14301 
14302 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14303   match(Set dst (NegF src));
14304 
14305   ins_cost(INSN_COST * 3);
14306   format %{ "fneg   $dst, $src" %}
14307 
14308   ins_encode %{
14309     __ fnegs(as_FloatRegister($dst$$reg),
14310              as_FloatRegister($src$$reg));
14311   %}
14312 
14313   ins_pipe(fp_uop_s);
14314 %}
14315 
14316 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14317   match(Set dst (NegD src));
14318 
14319   ins_cost(INSN_COST * 3);
14320   format %{ "fnegd   $dst, $src" %}
14321 
14322   ins_encode %{
14323     __ fnegd(as_FloatRegister($dst$$reg),
14324              as_FloatRegister($src$$reg));
14325   %}
14326 
14327   ins_pipe(fp_uop_d);
14328 %}
14329 
14330 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14331 %{
14332   match(Set dst (AbsI src));
14333 
14334   effect(KILL cr);
14335   ins_cost(INSN_COST * 2);
14336   format %{ "cmpw  $src, zr\n\t"
14337             "cnegw $dst, $src, Assembler::LT\t# int abs"
14338   %}
14339 
14340   ins_encode %{
14341     __ cmpw(as_Register($src$$reg), zr);
14342     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14343   %}
14344   ins_pipe(pipe_class_default);
14345 %}
14346 
14347 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14348 %{
14349   match(Set dst (AbsL src));
14350 
14351   effect(KILL cr);
14352   ins_cost(INSN_COST * 2);
14353   format %{ "cmp  $src, zr\n\t"
14354             "cneg $dst, $src, Assembler::LT\t# long abs"
14355   %}
14356 
14357   ins_encode %{
14358     __ cmp(as_Register($src$$reg), zr);
14359     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14360   %}
14361   ins_pipe(pipe_class_default);
14362 %}
14363 
14364 instruct absF_reg(vRegF dst, vRegF src) %{
14365   match(Set dst (AbsF src));
14366 
14367   ins_cost(INSN_COST * 3);
14368   format %{ "fabss   $dst, $src" %}
14369   ins_encode %{
14370     __ fabss(as_FloatRegister($dst$$reg),
14371              as_FloatRegister($src$$reg));
14372   %}
14373 
14374   ins_pipe(fp_uop_s);
14375 %}
14376 
14377 instruct absD_reg(vRegD dst, vRegD src) %{
14378   match(Set dst (AbsD src));
14379 
14380   ins_cost(INSN_COST * 3);
14381   format %{ "fabsd   $dst, $src" %}
14382   ins_encode %{
14383     __ fabsd(as_FloatRegister($dst$$reg),
14384              as_FloatRegister($src$$reg));
14385   %}
14386 
14387   ins_pipe(fp_uop_d);
14388 %}
14389 
14390 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14391   match(Set dst (AbsF (SubF src1 src2)));
14392 
14393   ins_cost(INSN_COST * 3);
14394   format %{ "fabds   $dst, $src1, $src2" %}
14395   ins_encode %{
14396     __ fabds(as_FloatRegister($dst$$reg),
14397              as_FloatRegister($src1$$reg),
14398              as_FloatRegister($src2$$reg));
14399   %}
14400 
14401   ins_pipe(fp_uop_s);
14402 %}
14403 
14404 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14405   match(Set dst (AbsD (SubD src1 src2)));
14406 
14407   ins_cost(INSN_COST * 3);
14408   format %{ "fabdd   $dst, $src1, $src2" %}
14409   ins_encode %{
14410     __ fabdd(as_FloatRegister($dst$$reg),
14411              as_FloatRegister($src1$$reg),
14412              as_FloatRegister($src2$$reg));
14413   %}
14414 
14415   ins_pipe(fp_uop_d);
14416 %}
14417 
14418 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14419   match(Set dst (SqrtD src));
14420 
14421   ins_cost(INSN_COST * 50);
14422   format %{ "fsqrtd  $dst, $src" %}
14423   ins_encode %{
14424     __ fsqrtd(as_FloatRegister($dst$$reg),
14425              as_FloatRegister($src$$reg));
14426   %}
14427 
14428   ins_pipe(fp_div_s);
14429 %}
14430 
14431 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14432   match(Set dst (SqrtF src));
14433 
14434   ins_cost(INSN_COST * 50);
14435   format %{ "fsqrts  $dst, $src" %}
14436   ins_encode %{
14437     __ fsqrts(as_FloatRegister($dst$$reg),
14438              as_FloatRegister($src$$reg));
14439   %}
14440 
14441   ins_pipe(fp_div_d);
14442 %}
14443 
14444 // Math.rint, floor, ceil
14445 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14446   match(Set dst (RoundDoubleMode src rmode));
14447   format %{ "frint  $dst, $src, $rmode" %}
14448   ins_encode %{
14449     switch ($rmode$$constant) {
14450       case RoundDoubleModeNode::rmode_rint:
14451         __ frintnd(as_FloatRegister($dst$$reg),
14452                    as_FloatRegister($src$$reg));
14453         break;
14454       case RoundDoubleModeNode::rmode_floor:
14455         __ frintmd(as_FloatRegister($dst$$reg),
14456                    as_FloatRegister($src$$reg));
14457         break;
14458       case RoundDoubleModeNode::rmode_ceil:
14459         __ frintpd(as_FloatRegister($dst$$reg),
14460                    as_FloatRegister($src$$reg));
14461         break;
14462     }
14463   %}
14464   ins_pipe(fp_uop_d);
14465 %}
14466 
14467 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14468   match(Set dst (CopySignD src1 (Binary src2 zero)));
14469   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14470   format %{ "CopySignD  $dst $src1 $src2" %}
14471   ins_encode %{
14472     FloatRegister dst = as_FloatRegister($dst$$reg),
14473                   src1 = as_FloatRegister($src1$$reg),
14474                   src2 = as_FloatRegister($src2$$reg),
14475                   zero = as_FloatRegister($zero$$reg);
14476     __ fnegd(dst, zero);
14477     __ bsl(dst, __ T8B, src2, src1);
14478   %}
14479   ins_pipe(fp_uop_d);
14480 %}
14481 
14482 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14483   match(Set dst (CopySignF src1 src2));
14484   effect(TEMP_DEF dst, USE src1, USE src2);
14485   format %{ "CopySignF  $dst $src1 $src2" %}
14486   ins_encode %{
14487     FloatRegister dst = as_FloatRegister($dst$$reg),
14488                   src1 = as_FloatRegister($src1$$reg),
14489                   src2 = as_FloatRegister($src2$$reg);
14490     __ movi(dst, __ T2S, 0x80, 24);
14491     __ bsl(dst, __ T8B, src2, src1);
14492   %}
14493   ins_pipe(fp_uop_d);
14494 %}
14495 
14496 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14497   match(Set dst (SignumD src (Binary zero one)));
14498   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14499   format %{ "signumD  $dst, $src" %}
14500   ins_encode %{
14501     FloatRegister src = as_FloatRegister($src$$reg),
14502                   dst = as_FloatRegister($dst$$reg),
14503                   zero = as_FloatRegister($zero$$reg),
14504                   one = as_FloatRegister($one$$reg);
14505     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14506     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14507     // Bit selection instruction gets bit from "one" for each enabled bit in
14508     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14509     // NaN the whole "src" will be copied because "dst" is zero. For all other
14510     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14511     // from "src", and all other bits are copied from 1.0.
14512     __ bsl(dst, __ T8B, one, src);
14513   %}
14514   ins_pipe(fp_uop_d);
14515 %}
14516 
14517 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14518   match(Set dst (SignumF src (Binary zero one)));
14519   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14520   format %{ "signumF  $dst, $src" %}
14521   ins_encode %{
14522     FloatRegister src = as_FloatRegister($src$$reg),
14523                   dst = as_FloatRegister($dst$$reg),
14524                   zero = as_FloatRegister($zero$$reg),
14525                   one = as_FloatRegister($one$$reg);
14526     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14527     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14528     // Bit selection instruction gets bit from "one" for each enabled bit in
14529     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14530     // NaN the whole "src" will be copied because "dst" is zero. For all other
14531     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14532     // from "src", and all other bits are copied from 1.0.
14533     __ bsl(dst, __ T8B, one, src);
14534   %}
14535   ins_pipe(fp_uop_d);
14536 %}
14537 
14538 instruct onspinwait() %{
14539   match(OnSpinWait);
14540   ins_cost(INSN_COST);
14541 
14542   format %{ "onspinwait" %}
14543 
14544   ins_encode %{
14545     __ spin_wait();
14546   %}
14547   ins_pipe(pipe_class_empty);
14548 %}
14549 
14550 // ============================================================================
14551 // Logical Instructions
14552 
14553 // Integer Logical Instructions
14554 
14555 // And Instructions
14556 
14557 
14558 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14559   match(Set dst (AndI src1 src2));
14560 
14561   format %{ "andw  $dst, $src1, $src2\t# int" %}
14562 
14563   ins_cost(INSN_COST);
14564   ins_encode %{
14565     __ andw(as_Register($dst$$reg),
14566             as_Register($src1$$reg),
14567             as_Register($src2$$reg));
14568   %}
14569 
14570   ins_pipe(ialu_reg_reg);
14571 %}
14572 
14573 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14574   match(Set dst (AndI src1 src2));
14575 
14576   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14577 
14578   ins_cost(INSN_COST);
14579   ins_encode %{
14580     __ andw(as_Register($dst$$reg),
14581             as_Register($src1$$reg),
14582             (uint64_t)($src2$$constant));
14583   %}
14584 
14585   ins_pipe(ialu_reg_imm);
14586 %}
14587 
14588 // Or Instructions
14589 
14590 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14591   match(Set dst (OrI src1 src2));
14592 
14593   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14594 
14595   ins_cost(INSN_COST);
14596   ins_encode %{
14597     __ orrw(as_Register($dst$$reg),
14598             as_Register($src1$$reg),
14599             as_Register($src2$$reg));
14600   %}
14601 
14602   ins_pipe(ialu_reg_reg);
14603 %}
14604 
14605 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14606   match(Set dst (OrI src1 src2));
14607 
14608   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14609 
14610   ins_cost(INSN_COST);
14611   ins_encode %{
14612     __ orrw(as_Register($dst$$reg),
14613             as_Register($src1$$reg),
14614             (uint64_t)($src2$$constant));
14615   %}
14616 
14617   ins_pipe(ialu_reg_imm);
14618 %}
14619 
14620 // Xor Instructions
14621 
14622 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14623   match(Set dst (XorI src1 src2));
14624 
14625   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14626 
14627   ins_cost(INSN_COST);
14628   ins_encode %{
14629     __ eorw(as_Register($dst$$reg),
14630             as_Register($src1$$reg),
14631             as_Register($src2$$reg));
14632   %}
14633 
14634   ins_pipe(ialu_reg_reg);
14635 %}
14636 
14637 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14638   match(Set dst (XorI src1 src2));
14639 
14640   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14641 
14642   ins_cost(INSN_COST);
14643   ins_encode %{
14644     __ eorw(as_Register($dst$$reg),
14645             as_Register($src1$$reg),
14646             (uint64_t)($src2$$constant));
14647   %}
14648 
14649   ins_pipe(ialu_reg_imm);
14650 %}
14651 
14652 // Long Logical Instructions
14653 // TODO
14654 
14655 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14656   match(Set dst (AndL src1 src2));
14657 
14658   format %{ "and  $dst, $src1, $src2\t# int" %}
14659 
14660   ins_cost(INSN_COST);
14661   ins_encode %{
14662     __ andr(as_Register($dst$$reg),
14663             as_Register($src1$$reg),
14664             as_Register($src2$$reg));
14665   %}
14666 
14667   ins_pipe(ialu_reg_reg);
14668 %}
14669 
14670 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14671   match(Set dst (AndL src1 src2));
14672 
14673   format %{ "and  $dst, $src1, $src2\t# int" %}
14674 
14675   ins_cost(INSN_COST);
14676   ins_encode %{
14677     __ andr(as_Register($dst$$reg),
14678             as_Register($src1$$reg),
14679             (uint64_t)($src2$$constant));
14680   %}
14681 
14682   ins_pipe(ialu_reg_imm);
14683 %}
14684 
14685 // Or Instructions
14686 
14687 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14688   match(Set dst (OrL src1 src2));
14689 
14690   format %{ "orr  $dst, $src1, $src2\t# int" %}
14691 
14692   ins_cost(INSN_COST);
14693   ins_encode %{
14694     __ orr(as_Register($dst$$reg),
14695            as_Register($src1$$reg),
14696            as_Register($src2$$reg));
14697   %}
14698 
14699   ins_pipe(ialu_reg_reg);
14700 %}
14701 
14702 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14703   match(Set dst (OrL src1 src2));
14704 
14705   format %{ "orr  $dst, $src1, $src2\t# int" %}
14706 
14707   ins_cost(INSN_COST);
14708   ins_encode %{
14709     __ orr(as_Register($dst$$reg),
14710            as_Register($src1$$reg),
14711            (uint64_t)($src2$$constant));
14712   %}
14713 
14714   ins_pipe(ialu_reg_imm);
14715 %}
14716 
14717 // Xor Instructions
14718 
14719 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14720   match(Set dst (XorL src1 src2));
14721 
14722   format %{ "eor  $dst, $src1, $src2\t# int" %}
14723 
14724   ins_cost(INSN_COST);
14725   ins_encode %{
14726     __ eor(as_Register($dst$$reg),
14727            as_Register($src1$$reg),
14728            as_Register($src2$$reg));
14729   %}
14730 
14731   ins_pipe(ialu_reg_reg);
14732 %}
14733 
14734 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14735   match(Set dst (XorL src1 src2));
14736 
14737   ins_cost(INSN_COST);
14738   format %{ "eor  $dst, $src1, $src2\t# int" %}
14739 
14740   ins_encode %{
14741     __ eor(as_Register($dst$$reg),
14742            as_Register($src1$$reg),
14743            (uint64_t)($src2$$constant));
14744   %}
14745 
14746   ins_pipe(ialu_reg_imm);
14747 %}
14748 
14749 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14750 %{
14751   match(Set dst (ConvI2L src));
14752 
14753   ins_cost(INSN_COST);
14754   format %{ "sxtw  $dst, $src\t# i2l" %}
14755   ins_encode %{
14756     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14757   %}
14758   ins_pipe(ialu_reg_shift);
14759 %}
14760 
14761 // this pattern occurs in bigmath arithmetic
14762 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14763 %{
14764   match(Set dst (AndL (ConvI2L src) mask));
14765 
14766   ins_cost(INSN_COST);
14767   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14768   ins_encode %{
14769     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14770   %}
14771 
14772   ins_pipe(ialu_reg_shift);
14773 %}
14774 
14775 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14776   match(Set dst (ConvL2I src));
14777 
14778   ins_cost(INSN_COST);
14779   format %{ "movw  $dst, $src \t// l2i" %}
14780 
14781   ins_encode %{
14782     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14783   %}
14784 
14785   ins_pipe(ialu_reg);
14786 %}
14787 
14788 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14789 %{
14790   match(Set dst (Conv2B src));
14791   effect(KILL cr);
14792 
14793   format %{
14794     "cmpw $src, zr\n\t"
14795     "cset $dst, ne"
14796   %}
14797 
14798   ins_encode %{
14799     __ cmpw(as_Register($src$$reg), zr);
14800     __ cset(as_Register($dst$$reg), Assembler::NE);
14801   %}
14802 
14803   ins_pipe(ialu_reg);
14804 %}
14805 
14806 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14807 %{
14808   match(Set dst (Conv2B src));
14809   effect(KILL cr);
14810 
14811   format %{
14812     "cmp  $src, zr\n\t"
14813     "cset $dst, ne"
14814   %}
14815 
14816   ins_encode %{
14817     __ cmp(as_Register($src$$reg), zr);
14818     __ cset(as_Register($dst$$reg), Assembler::NE);
14819   %}
14820 
14821   ins_pipe(ialu_reg);
14822 %}
14823 
14824 instruct convD2F_reg(vRegF dst, vRegD src) %{
14825   match(Set dst (ConvD2F src));
14826 
14827   ins_cost(INSN_COST * 5);
14828   format %{ "fcvtd  $dst, $src \t// d2f" %}
14829 
14830   ins_encode %{
14831     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14832   %}
14833 
14834   ins_pipe(fp_d2f);
14835 %}
14836 
14837 instruct convF2D_reg(vRegD dst, vRegF src) %{
14838   match(Set dst (ConvF2D src));
14839 
14840   ins_cost(INSN_COST * 5);
14841   format %{ "fcvts  $dst, $src \t// f2d" %}
14842 
14843   ins_encode %{
14844     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14845   %}
14846 
14847   ins_pipe(fp_f2d);
14848 %}
14849 
14850 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14851   match(Set dst (ConvF2I src));
14852 
14853   ins_cost(INSN_COST * 5);
14854   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14855 
14856   ins_encode %{
14857     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14858   %}
14859 
14860   ins_pipe(fp_f2i);
14861 %}
14862 
14863 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14864   match(Set dst (ConvF2L src));
14865 
14866   ins_cost(INSN_COST * 5);
14867   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14868 
14869   ins_encode %{
14870     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14871   %}
14872 
14873   ins_pipe(fp_f2l);
14874 %}
14875 
14876 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14877   match(Set dst (ConvI2F src));
14878 
14879   ins_cost(INSN_COST * 5);
14880   format %{ "scvtfws  $dst, $src \t// i2f" %}
14881 
14882   ins_encode %{
14883     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14884   %}
14885 
14886   ins_pipe(fp_i2f);
14887 %}
14888 
14889 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14890   match(Set dst (ConvL2F src));
14891 
14892   ins_cost(INSN_COST * 5);
14893   format %{ "scvtfs  $dst, $src \t// l2f" %}
14894 
14895   ins_encode %{
14896     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14897   %}
14898 
14899   ins_pipe(fp_l2f);
14900 %}
14901 
14902 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14903   match(Set dst (ConvD2I src));
14904 
14905   ins_cost(INSN_COST * 5);
14906   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14907 
14908   ins_encode %{
14909     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14910   %}
14911 
14912   ins_pipe(fp_d2i);
14913 %}
14914 
14915 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14916   match(Set dst (ConvD2L src));
14917 
14918   ins_cost(INSN_COST * 5);
14919   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14920 
14921   ins_encode %{
14922     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14923   %}
14924 
14925   ins_pipe(fp_d2l);
14926 %}
14927 
14928 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14929   match(Set dst (ConvI2D src));
14930 
14931   ins_cost(INSN_COST * 5);
14932   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14933 
14934   ins_encode %{
14935     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14936   %}
14937 
14938   ins_pipe(fp_i2d);
14939 %}
14940 
14941 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14942   match(Set dst (ConvL2D src));
14943 
14944   ins_cost(INSN_COST * 5);
14945   format %{ "scvtfd  $dst, $src \t// l2d" %}
14946 
14947   ins_encode %{
14948     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14949   %}
14950 
14951   ins_pipe(fp_l2d);
14952 %}
14953 
14954 // stack <-> reg and reg <-> reg shuffles with no conversion
14955 
14956 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14957 
14958   match(Set dst (MoveF2I src));
14959 
14960   effect(DEF dst, USE src);
14961 
14962   ins_cost(4 * INSN_COST);
14963 
14964   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14965 
14966   ins_encode %{
14967     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14968   %}
14969 
14970   ins_pipe(iload_reg_reg);
14971 
14972 %}
14973 
14974 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14975 
14976   match(Set dst (MoveI2F src));
14977 
14978   effect(DEF dst, USE src);
14979 
14980   ins_cost(4 * INSN_COST);
14981 
14982   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14983 
14984   ins_encode %{
14985     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14986   %}
14987 
14988   ins_pipe(pipe_class_memory);
14989 
14990 %}
14991 
14992 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14993 
14994   match(Set dst (MoveD2L src));
14995 
14996   effect(DEF dst, USE src);
14997 
14998   ins_cost(4 * INSN_COST);
14999 
15000   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15001 
15002   ins_encode %{
15003     __ ldr($dst$$Register, Address(sp, $src$$disp));
15004   %}
15005 
15006   ins_pipe(iload_reg_reg);
15007 
15008 %}
15009 
15010 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15011 
15012   match(Set dst (MoveL2D src));
15013 
15014   effect(DEF dst, USE src);
15015 
15016   ins_cost(4 * INSN_COST);
15017 
15018   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15019 
15020   ins_encode %{
15021     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15022   %}
15023 
15024   ins_pipe(pipe_class_memory);
15025 
15026 %}
15027 
15028 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15029 
15030   match(Set dst (MoveF2I src));
15031 
15032   effect(DEF dst, USE src);
15033 
15034   ins_cost(INSN_COST);
15035 
15036   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15037 
15038   ins_encode %{
15039     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15040   %}
15041 
15042   ins_pipe(pipe_class_memory);
15043 
15044 %}
15045 
15046 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15047 
15048   match(Set dst (MoveI2F src));
15049 
15050   effect(DEF dst, USE src);
15051 
15052   ins_cost(INSN_COST);
15053 
15054   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15055 
15056   ins_encode %{
15057     __ strw($src$$Register, Address(sp, $dst$$disp));
15058   %}
15059 
15060   ins_pipe(istore_reg_reg);
15061 
15062 %}
15063 
15064 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15065 
15066   match(Set dst (MoveD2L src));
15067 
15068   effect(DEF dst, USE src);
15069 
15070   ins_cost(INSN_COST);
15071 
15072   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15073 
15074   ins_encode %{
15075     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15076   %}
15077 
15078   ins_pipe(pipe_class_memory);
15079 
15080 %}
15081 
15082 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15083 
15084   match(Set dst (MoveL2D src));
15085 
15086   effect(DEF dst, USE src);
15087 
15088   ins_cost(INSN_COST);
15089 
15090   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15091 
15092   ins_encode %{
15093     __ str($src$$Register, Address(sp, $dst$$disp));
15094   %}
15095 
15096   ins_pipe(istore_reg_reg);
15097 
15098 %}
15099 
15100 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15101 
15102   match(Set dst (MoveF2I src));
15103 
15104   effect(DEF dst, USE src);
15105 
15106   ins_cost(INSN_COST);
15107 
15108   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15109 
15110   ins_encode %{
15111     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15112   %}
15113 
15114   ins_pipe(fp_f2i);
15115 
15116 %}
15117 
15118 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15119 
15120   match(Set dst (MoveI2F src));
15121 
15122   effect(DEF dst, USE src);
15123 
15124   ins_cost(INSN_COST);
15125 
15126   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15127 
15128   ins_encode %{
15129     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15130   %}
15131 
15132   ins_pipe(fp_i2f);
15133 
15134 %}
15135 
15136 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15137 
15138   match(Set dst (MoveD2L src));
15139 
15140   effect(DEF dst, USE src);
15141 
15142   ins_cost(INSN_COST);
15143 
15144   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15145 
15146   ins_encode %{
15147     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15148   %}
15149 
15150   ins_pipe(fp_d2l);
15151 
15152 %}
15153 
15154 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15155 
15156   match(Set dst (MoveL2D src));
15157 
15158   effect(DEF dst, USE src);
15159 
15160   ins_cost(INSN_COST);
15161 
15162   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15163 
15164   ins_encode %{
15165     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15166   %}
15167 
15168   ins_pipe(fp_l2d);
15169 
15170 %}
15171 
15172 // ============================================================================
15173 // clearing of an array
15174 
15175 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15176 %{
15177   match(Set dummy (ClearArray cnt base));
15178   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15179 
15180   ins_cost(4 * INSN_COST);
15181   format %{ "ClearArray $cnt, $base" %}
15182 
15183   ins_encode %{
15184     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15185     if (tpc == NULL) {
15186       ciEnv::current()->record_failure("CodeCache is full");
15187       return;
15188     }
15189   %}
15190 
15191   ins_pipe(pipe_class_memory);
15192 %}
15193 
15194 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15195 %{
15196   predicate((uint64_t)n->in(2)->get_long()
15197             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15198   match(Set dummy (ClearArray cnt base));
15199   effect(TEMP temp, USE_KILL base, KILL cr);
15200 
15201   ins_cost(4 * INSN_COST);
15202   format %{ "ClearArray $cnt, $base" %}
15203 
15204   ins_encode %{
15205     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15206   %}
15207 
15208   ins_pipe(pipe_class_memory);
15209 %}
15210 
15211 // ============================================================================
15212 // Overflow Math Instructions
15213 
15214 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15215 %{
15216   match(Set cr (OverflowAddI op1 op2));
15217 
15218   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15219   ins_cost(INSN_COST);
15220   ins_encode %{
15221     __ cmnw($op1$$Register, $op2$$Register);
15222   %}
15223 
15224   ins_pipe(icmp_reg_reg);
15225 %}
15226 
15227 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15228 %{
15229   match(Set cr (OverflowAddI op1 op2));
15230 
15231   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15232   ins_cost(INSN_COST);
15233   ins_encode %{
15234     __ cmnw($op1$$Register, $op2$$constant);
15235   %}
15236 
15237   ins_pipe(icmp_reg_imm);
15238 %}
15239 
15240 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15241 %{
15242   match(Set cr (OverflowAddL op1 op2));
15243 
15244   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15245   ins_cost(INSN_COST);
15246   ins_encode %{
15247     __ cmn($op1$$Register, $op2$$Register);
15248   %}
15249 
15250   ins_pipe(icmp_reg_reg);
15251 %}
15252 
15253 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15254 %{
15255   match(Set cr (OverflowAddL op1 op2));
15256 
15257   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15258   ins_cost(INSN_COST);
15259   ins_encode %{
15260     __ cmn($op1$$Register, $op2$$constant);
15261   %}
15262 
15263   ins_pipe(icmp_reg_imm);
15264 %}
15265 
15266 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15267 %{
15268   match(Set cr (OverflowSubI op1 op2));
15269 
15270   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15271   ins_cost(INSN_COST);
15272   ins_encode %{
15273     __ cmpw($op1$$Register, $op2$$Register);
15274   %}
15275 
15276   ins_pipe(icmp_reg_reg);
15277 %}
15278 
15279 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15280 %{
15281   match(Set cr (OverflowSubI op1 op2));
15282 
15283   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15284   ins_cost(INSN_COST);
15285   ins_encode %{
15286     __ cmpw($op1$$Register, $op2$$constant);
15287   %}
15288 
15289   ins_pipe(icmp_reg_imm);
15290 %}
15291 
15292 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15293 %{
15294   match(Set cr (OverflowSubL op1 op2));
15295 
15296   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15297   ins_cost(INSN_COST);
15298   ins_encode %{
15299     __ cmp($op1$$Register, $op2$$Register);
15300   %}
15301 
15302   ins_pipe(icmp_reg_reg);
15303 %}
15304 
15305 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15306 %{
15307   match(Set cr (OverflowSubL op1 op2));
15308 
15309   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15310   ins_cost(INSN_COST);
15311   ins_encode %{
15312     __ subs(zr, $op1$$Register, $op2$$constant);
15313   %}
15314 
15315   ins_pipe(icmp_reg_imm);
15316 %}
15317 
15318 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15319 %{
15320   match(Set cr (OverflowSubI zero op1));
15321 
15322   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15323   ins_cost(INSN_COST);
15324   ins_encode %{
15325     __ cmpw(zr, $op1$$Register);
15326   %}
15327 
15328   ins_pipe(icmp_reg_imm);
15329 %}
15330 
15331 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15332 %{
15333   match(Set cr (OverflowSubL zero op1));
15334 
15335   format %{ "cmp   zr, $op1\t# overflow check long" %}
15336   ins_cost(INSN_COST);
15337   ins_encode %{
15338     __ cmp(zr, $op1$$Register);
15339   %}
15340 
15341   ins_pipe(icmp_reg_imm);
15342 %}
15343 
15344 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15345 %{
15346   match(Set cr (OverflowMulI op1 op2));
15347 
15348   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15349             "cmp   rscratch1, rscratch1, sxtw\n\t"
15350             "movw  rscratch1, #0x80000000\n\t"
15351             "cselw rscratch1, rscratch1, zr, NE\n\t"
15352             "cmpw  rscratch1, #1" %}
15353   ins_cost(5 * INSN_COST);
15354   ins_encode %{
15355     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15356     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15357     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15358     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15359     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15360   %}
15361 
15362   ins_pipe(pipe_slow);
15363 %}
15364 
15365 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15366 %{
15367   match(If cmp (OverflowMulI op1 op2));
15368   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15369             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15370   effect(USE labl, KILL cr);
15371 
15372   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15373             "cmp   rscratch1, rscratch1, sxtw\n\t"
15374             "b$cmp   $labl" %}
15375   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15376   ins_encode %{
15377     Label* L = $labl$$label;
15378     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15379     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15380     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15381     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15382   %}
15383 
15384   ins_pipe(pipe_serial);
15385 %}
15386 
15387 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15388 %{
15389   match(Set cr (OverflowMulL op1 op2));
15390 
15391   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15392             "smulh rscratch2, $op1, $op2\n\t"
15393             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15394             "movw  rscratch1, #0x80000000\n\t"
15395             "cselw rscratch1, rscratch1, zr, NE\n\t"
15396             "cmpw  rscratch1, #1" %}
15397   ins_cost(6 * INSN_COST);
15398   ins_encode %{
15399     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15400     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15401     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15402     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15403     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15404     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15405   %}
15406 
15407   ins_pipe(pipe_slow);
15408 %}
15409 
15410 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15411 %{
15412   match(If cmp (OverflowMulL op1 op2));
15413   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15414             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15415   effect(USE labl, KILL cr);
15416 
15417   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15418             "smulh rscratch2, $op1, $op2\n\t"
15419             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15420             "b$cmp $labl" %}
15421   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15422   ins_encode %{
15423     Label* L = $labl$$label;
15424     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15425     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15426     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15427     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15428     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15429   %}
15430 
15431   ins_pipe(pipe_serial);
15432 %}
15433 
15434 // ============================================================================
15435 // Compare Instructions
15436 
15437 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15438 %{
15439   match(Set cr (CmpI op1 op2));
15440 
15441   effect(DEF cr, USE op1, USE op2);
15442 
15443   ins_cost(INSN_COST);
15444   format %{ "cmpw  $op1, $op2" %}
15445 
15446   ins_encode(aarch64_enc_cmpw(op1, op2));
15447 
15448   ins_pipe(icmp_reg_reg);
15449 %}
15450 
15451 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15452 %{
15453   match(Set cr (CmpI op1 zero));
15454 
15455   effect(DEF cr, USE op1);
15456 
15457   ins_cost(INSN_COST);
15458   format %{ "cmpw $op1, 0" %}
15459 
15460   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15461 
15462   ins_pipe(icmp_reg_imm);
15463 %}
15464 
15465 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15466 %{
15467   match(Set cr (CmpI op1 op2));
15468 
15469   effect(DEF cr, USE op1);
15470 
15471   ins_cost(INSN_COST);
15472   format %{ "cmpw  $op1, $op2" %}
15473 
15474   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15475 
15476   ins_pipe(icmp_reg_imm);
15477 %}
15478 
15479 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15480 %{
15481   match(Set cr (CmpI op1 op2));
15482 
15483   effect(DEF cr, USE op1);
15484 
15485   ins_cost(INSN_COST * 2);
15486   format %{ "cmpw  $op1, $op2" %}
15487 
15488   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15489 
15490   ins_pipe(icmp_reg_imm);
15491 %}
15492 
15493 // Unsigned compare Instructions; really, same as signed compare
15494 // except it should only be used to feed an If or a CMovI which takes a
15495 // cmpOpU.
15496 
15497 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15498 %{
15499   match(Set cr (CmpU op1 op2));
15500 
15501   effect(DEF cr, USE op1, USE op2);
15502 
15503   ins_cost(INSN_COST);
15504   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15505 
15506   ins_encode(aarch64_enc_cmpw(op1, op2));
15507 
15508   ins_pipe(icmp_reg_reg);
15509 %}
15510 
15511 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15512 %{
15513   match(Set cr (CmpU op1 zero));
15514 
15515   effect(DEF cr, USE op1);
15516 
15517   ins_cost(INSN_COST);
15518   format %{ "cmpw $op1, #0\t# unsigned" %}
15519 
15520   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15521 
15522   ins_pipe(icmp_reg_imm);
15523 %}
15524 
15525 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15526 %{
15527   match(Set cr (CmpU op1 op2));
15528 
15529   effect(DEF cr, USE op1);
15530 
15531   ins_cost(INSN_COST);
15532   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15533 
15534   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15535 
15536   ins_pipe(icmp_reg_imm);
15537 %}
15538 
15539 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15540 %{
15541   match(Set cr (CmpU op1 op2));
15542 
15543   effect(DEF cr, USE op1);
15544 
15545   ins_cost(INSN_COST * 2);
15546   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15547 
15548   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15549 
15550   ins_pipe(icmp_reg_imm);
15551 %}
15552 
15553 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15554 %{
15555   match(Set cr (CmpL op1 op2));
15556 
15557   effect(DEF cr, USE op1, USE op2);
15558 
15559   ins_cost(INSN_COST);
15560   format %{ "cmp  $op1, $op2" %}
15561 
15562   ins_encode(aarch64_enc_cmp(op1, op2));
15563 
15564   ins_pipe(icmp_reg_reg);
15565 %}
15566 
15567 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15568 %{
15569   match(Set cr (CmpL op1 zero));
15570 
15571   effect(DEF cr, USE op1);
15572 
15573   ins_cost(INSN_COST);
15574   format %{ "tst  $op1" %}
15575 
15576   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15577 
15578   ins_pipe(icmp_reg_imm);
15579 %}
15580 
15581 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15582 %{
15583   match(Set cr (CmpL op1 op2));
15584 
15585   effect(DEF cr, USE op1);
15586 
15587   ins_cost(INSN_COST);
15588   format %{ "cmp  $op1, $op2" %}
15589 
15590   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15591 
15592   ins_pipe(icmp_reg_imm);
15593 %}
15594 
15595 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15596 %{
15597   match(Set cr (CmpL op1 op2));
15598 
15599   effect(DEF cr, USE op1);
15600 
15601   ins_cost(INSN_COST * 2);
15602   format %{ "cmp  $op1, $op2" %}
15603 
15604   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15605 
15606   ins_pipe(icmp_reg_imm);
15607 %}
15608 
15609 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15610 %{
15611   match(Set cr (CmpUL op1 op2));
15612 
15613   effect(DEF cr, USE op1, USE op2);
15614 
15615   ins_cost(INSN_COST);
15616   format %{ "cmp  $op1, $op2" %}
15617 
15618   ins_encode(aarch64_enc_cmp(op1, op2));
15619 
15620   ins_pipe(icmp_reg_reg);
15621 %}
15622 
15623 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15624 %{
15625   match(Set cr (CmpUL op1 zero));
15626 
15627   effect(DEF cr, USE op1);
15628 
15629   ins_cost(INSN_COST);
15630   format %{ "tst  $op1" %}
15631 
15632   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15633 
15634   ins_pipe(icmp_reg_imm);
15635 %}
15636 
15637 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15638 %{
15639   match(Set cr (CmpUL op1 op2));
15640 
15641   effect(DEF cr, USE op1);
15642 
15643   ins_cost(INSN_COST);
15644   format %{ "cmp  $op1, $op2" %}
15645 
15646   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15647 
15648   ins_pipe(icmp_reg_imm);
15649 %}
15650 
15651 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15652 %{
15653   match(Set cr (CmpUL op1 op2));
15654 
15655   effect(DEF cr, USE op1);
15656 
15657   ins_cost(INSN_COST * 2);
15658   format %{ "cmp  $op1, $op2" %}
15659 
15660   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15661 
15662   ins_pipe(icmp_reg_imm);
15663 %}
15664 
15665 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15666 %{
15667   match(Set cr (CmpP op1 op2));
15668 
15669   effect(DEF cr, USE op1, USE op2);
15670 
15671   ins_cost(INSN_COST);
15672   format %{ "cmp  $op1, $op2\t // ptr" %}
15673 
15674   ins_encode(aarch64_enc_cmpp(op1, op2));
15675 
15676   ins_pipe(icmp_reg_reg);
15677 %}
15678 
15679 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15680 %{
15681   match(Set cr (CmpN op1 op2));
15682 
15683   effect(DEF cr, USE op1, USE op2);
15684 
15685   ins_cost(INSN_COST);
15686   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15687 
15688   ins_encode(aarch64_enc_cmpn(op1, op2));
15689 
15690   ins_pipe(icmp_reg_reg);
15691 %}
15692 
15693 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15694 %{
15695   match(Set cr (CmpP op1 zero));
15696 
15697   effect(DEF cr, USE op1, USE zero);
15698 
15699   ins_cost(INSN_COST);
15700   format %{ "cmp  $op1, 0\t // ptr" %}
15701 
15702   ins_encode(aarch64_enc_testp(op1));
15703 
15704   ins_pipe(icmp_reg_imm);
15705 %}
15706 
15707 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15708 %{
15709   match(Set cr (CmpN op1 zero));
15710 
15711   effect(DEF cr, USE op1, USE zero);
15712 
15713   ins_cost(INSN_COST);
15714   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15715 
15716   ins_encode(aarch64_enc_testn(op1));
15717 
15718   ins_pipe(icmp_reg_imm);
15719 %}
15720 
15721 // FP comparisons
15722 //
15723 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15724 // using normal cmpOp. See declaration of rFlagsReg for details.
15725 
15726 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15727 %{
15728   match(Set cr (CmpF src1 src2));
15729 
15730   ins_cost(3 * INSN_COST);
15731   format %{ "fcmps $src1, $src2" %}
15732 
15733   ins_encode %{
15734     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15735   %}
15736 
15737   ins_pipe(pipe_class_compare);
15738 %}
15739 
15740 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15741 %{
15742   match(Set cr (CmpF src1 src2));
15743 
15744   ins_cost(3 * INSN_COST);
15745   format %{ "fcmps $src1, 0.0" %}
15746 
15747   ins_encode %{
15748     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15749   %}
15750 
15751   ins_pipe(pipe_class_compare);
15752 %}
15753 // FROM HERE
15754 
15755 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15756 %{
15757   match(Set cr (CmpD src1 src2));
15758 
15759   ins_cost(3 * INSN_COST);
15760   format %{ "fcmpd $src1, $src2" %}
15761 
15762   ins_encode %{
15763     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15764   %}
15765 
15766   ins_pipe(pipe_class_compare);
15767 %}
15768 
15769 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15770 %{
15771   match(Set cr (CmpD src1 src2));
15772 
15773   ins_cost(3 * INSN_COST);
15774   format %{ "fcmpd $src1, 0.0" %}
15775 
15776   ins_encode %{
15777     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15778   %}
15779 
15780   ins_pipe(pipe_class_compare);
15781 %}
15782 
15783 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15784 %{
15785   match(Set dst (CmpF3 src1 src2));
15786   effect(KILL cr);
15787 
15788   ins_cost(5 * INSN_COST);
15789   format %{ "fcmps $src1, $src2\n\t"
15790             "csinvw($dst, zr, zr, eq\n\t"
15791             "csnegw($dst, $dst, $dst, lt)"
15792   %}
15793 
15794   ins_encode %{
15795     Label done;
15796     FloatRegister s1 = as_FloatRegister($src1$$reg);
15797     FloatRegister s2 = as_FloatRegister($src2$$reg);
15798     Register d = as_Register($dst$$reg);
15799     __ fcmps(s1, s2);
15800     // installs 0 if EQ else -1
15801     __ csinvw(d, zr, zr, Assembler::EQ);
15802     // keeps -1 if less or unordered else installs 1
15803     __ csnegw(d, d, d, Assembler::LT);
15804     __ bind(done);
15805   %}
15806 
15807   ins_pipe(pipe_class_default);
15808 
15809 %}
15810 
15811 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15812 %{
15813   match(Set dst (CmpD3 src1 src2));
15814   effect(KILL cr);
15815 
15816   ins_cost(5 * INSN_COST);
15817   format %{ "fcmpd $src1, $src2\n\t"
15818             "csinvw($dst, zr, zr, eq\n\t"
15819             "csnegw($dst, $dst, $dst, lt)"
15820   %}
15821 
15822   ins_encode %{
15823     Label done;
15824     FloatRegister s1 = as_FloatRegister($src1$$reg);
15825     FloatRegister s2 = as_FloatRegister($src2$$reg);
15826     Register d = as_Register($dst$$reg);
15827     __ fcmpd(s1, s2);
15828     // installs 0 if EQ else -1
15829     __ csinvw(d, zr, zr, Assembler::EQ);
15830     // keeps -1 if less or unordered else installs 1
15831     __ csnegw(d, d, d, Assembler::LT);
15832     __ bind(done);
15833   %}
15834   ins_pipe(pipe_class_default);
15835 
15836 %}
15837 
15838 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15839 %{
15840   match(Set dst (CmpF3 src1 zero));
15841   effect(KILL cr);
15842 
15843   ins_cost(5 * INSN_COST);
15844   format %{ "fcmps $src1, 0.0\n\t"
15845             "csinvw($dst, zr, zr, eq\n\t"
15846             "csnegw($dst, $dst, $dst, lt)"
15847   %}
15848 
15849   ins_encode %{
15850     Label done;
15851     FloatRegister s1 = as_FloatRegister($src1$$reg);
15852     Register d = as_Register($dst$$reg);
15853     __ fcmps(s1, 0.0);
15854     // installs 0 if EQ else -1
15855     __ csinvw(d, zr, zr, Assembler::EQ);
15856     // keeps -1 if less or unordered else installs 1
15857     __ csnegw(d, d, d, Assembler::LT);
15858     __ bind(done);
15859   %}
15860 
15861   ins_pipe(pipe_class_default);
15862 
15863 %}
15864 
15865 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15866 %{
15867   match(Set dst (CmpD3 src1 zero));
15868   effect(KILL cr);
15869 
15870   ins_cost(5 * INSN_COST);
15871   format %{ "fcmpd $src1, 0.0\n\t"
15872             "csinvw($dst, zr, zr, eq\n\t"
15873             "csnegw($dst, $dst, $dst, lt)"
15874   %}
15875 
15876   ins_encode %{
15877     Label done;
15878     FloatRegister s1 = as_FloatRegister($src1$$reg);
15879     Register d = as_Register($dst$$reg);
15880     __ fcmpd(s1, 0.0);
15881     // installs 0 if EQ else -1
15882     __ csinvw(d, zr, zr, Assembler::EQ);
15883     // keeps -1 if less or unordered else installs 1
15884     __ csnegw(d, d, d, Assembler::LT);
15885     __ bind(done);
15886   %}
15887   ins_pipe(pipe_class_default);
15888 
15889 %}
15890 
15891 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15892 %{
15893   match(Set dst (CmpLTMask p q));
15894   effect(KILL cr);
15895 
15896   ins_cost(3 * INSN_COST);
15897 
15898   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15899             "csetw $dst, lt\n\t"
15900             "subw $dst, zr, $dst"
15901   %}
15902 
15903   ins_encode %{
15904     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15905     __ csetw(as_Register($dst$$reg), Assembler::LT);
15906     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15907   %}
15908 
15909   ins_pipe(ialu_reg_reg);
15910 %}
15911 
15912 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15913 %{
15914   match(Set dst (CmpLTMask src zero));
15915   effect(KILL cr);
15916 
15917   ins_cost(INSN_COST);
15918 
15919   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15920 
15921   ins_encode %{
15922     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15923   %}
15924 
15925   ins_pipe(ialu_reg_shift);
15926 %}
15927 
15928 // ============================================================================
15929 // Max and Min
15930 
15931 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15932 %{
15933   effect( DEF dst, USE src1, USE src2, USE cr );
15934 
15935   ins_cost(INSN_COST * 2);
15936   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15937 
15938   ins_encode %{
15939     __ cselw(as_Register($dst$$reg),
15940              as_Register($src1$$reg),
15941              as_Register($src2$$reg),
15942              Assembler::LT);
15943   %}
15944 
15945   ins_pipe(icond_reg_reg);
15946 %}
15947 
15948 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15949 %{
15950   match(Set dst (MinI src1 src2));
15951   ins_cost(INSN_COST * 3);
15952 
15953   expand %{
15954     rFlagsReg cr;
15955     compI_reg_reg(cr, src1, src2);
15956     cmovI_reg_reg_lt(dst, src1, src2, cr);
15957   %}
15958 
15959 %}
15960 // FROM HERE
15961 
15962 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15963 %{
15964   effect( DEF dst, USE src1, USE src2, USE cr );
15965 
15966   ins_cost(INSN_COST * 2);
15967   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15968 
15969   ins_encode %{
15970     __ cselw(as_Register($dst$$reg),
15971              as_Register($src1$$reg),
15972              as_Register($src2$$reg),
15973              Assembler::GT);
15974   %}
15975 
15976   ins_pipe(icond_reg_reg);
15977 %}
15978 
15979 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15980 %{
15981   match(Set dst (MaxI src1 src2));
15982   ins_cost(INSN_COST * 3);
15983   expand %{
15984     rFlagsReg cr;
15985     compI_reg_reg(cr, src1, src2);
15986     cmovI_reg_reg_gt(dst, src1, src2, cr);
15987   %}
15988 %}
15989 
15990 // ============================================================================
15991 // Branch Instructions
15992 
15993 // Direct Branch.
15994 instruct branch(label lbl)
15995 %{
15996   match(Goto);
15997 
15998   effect(USE lbl);
15999 
16000   ins_cost(BRANCH_COST);
16001   format %{ "b  $lbl" %}
16002 
16003   ins_encode(aarch64_enc_b(lbl));
16004 
16005   ins_pipe(pipe_branch);
16006 %}
16007 
16008 // Conditional Near Branch
16009 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16010 %{
16011   // Same match rule as `branchConFar'.
16012   match(If cmp cr);
16013 
16014   effect(USE lbl);
16015 
16016   ins_cost(BRANCH_COST);
16017   // If set to 1 this indicates that the current instruction is a
16018   // short variant of a long branch. This avoids using this
16019   // instruction in first-pass matching. It will then only be used in
16020   // the `Shorten_branches' pass.
16021   // ins_short_branch(1);
16022   format %{ "b$cmp  $lbl" %}
16023 
16024   ins_encode(aarch64_enc_br_con(cmp, lbl));
16025 
16026   ins_pipe(pipe_branch_cond);
16027 %}
16028 
16029 // Conditional Near Branch Unsigned
16030 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16031 %{
16032   // Same match rule as `branchConFar'.
16033   match(If cmp cr);
16034 
16035   effect(USE lbl);
16036 
16037   ins_cost(BRANCH_COST);
16038   // If set to 1 this indicates that the current instruction is a
16039   // short variant of a long branch. This avoids using this
16040   // instruction in first-pass matching. It will then only be used in
16041   // the `Shorten_branches' pass.
16042   // ins_short_branch(1);
16043   format %{ "b$cmp  $lbl\t# unsigned" %}
16044 
16045   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16046 
16047   ins_pipe(pipe_branch_cond);
16048 %}
16049 
16050 // Make use of CBZ and CBNZ.  These instructions, as well as being
16051 // shorter than (cmp; branch), have the additional benefit of not
16052 // killing the flags.
16053 
16054 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16055   match(If cmp (CmpI op1 op2));
16056   effect(USE labl);
16057 
16058   ins_cost(BRANCH_COST);
16059   format %{ "cbw$cmp   $op1, $labl" %}
16060   ins_encode %{
16061     Label* L = $labl$$label;
16062     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16063     if (cond == Assembler::EQ)
16064       __ cbzw($op1$$Register, *L);
16065     else
16066       __ cbnzw($op1$$Register, *L);
16067   %}
16068   ins_pipe(pipe_cmp_branch);
16069 %}
16070 
16071 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16072   match(If cmp (CmpL op1 op2));
16073   effect(USE labl);
16074 
16075   ins_cost(BRANCH_COST);
16076   format %{ "cb$cmp   $op1, $labl" %}
16077   ins_encode %{
16078     Label* L = $labl$$label;
16079     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16080     if (cond == Assembler::EQ)
16081       __ cbz($op1$$Register, *L);
16082     else
16083       __ cbnz($op1$$Register, *L);
16084   %}
16085   ins_pipe(pipe_cmp_branch);
16086 %}
16087 
16088 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16089   match(If cmp (CmpP op1 op2));
16090   effect(USE labl);
16091 
16092   ins_cost(BRANCH_COST);
16093   format %{ "cb$cmp   $op1, $labl" %}
16094   ins_encode %{
16095     Label* L = $labl$$label;
16096     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16097     if (cond == Assembler::EQ)
16098       __ cbz($op1$$Register, *L);
16099     else
16100       __ cbnz($op1$$Register, *L);
16101   %}
16102   ins_pipe(pipe_cmp_branch);
16103 %}
16104 
16105 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16106   match(If cmp (CmpN op1 op2));
16107   effect(USE labl);
16108 
16109   ins_cost(BRANCH_COST);
16110   format %{ "cbw$cmp   $op1, $labl" %}
16111   ins_encode %{
16112     Label* L = $labl$$label;
16113     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16114     if (cond == Assembler::EQ)
16115       __ cbzw($op1$$Register, *L);
16116     else
16117       __ cbnzw($op1$$Register, *L);
16118   %}
16119   ins_pipe(pipe_cmp_branch);
16120 %}
16121 
16122 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16123   match(If cmp (CmpP (DecodeN oop) zero));
16124   effect(USE labl);
16125 
16126   ins_cost(BRANCH_COST);
16127   format %{ "cb$cmp   $oop, $labl" %}
16128   ins_encode %{
16129     Label* L = $labl$$label;
16130     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16131     if (cond == Assembler::EQ)
16132       __ cbzw($oop$$Register, *L);
16133     else
16134       __ cbnzw($oop$$Register, *L);
16135   %}
16136   ins_pipe(pipe_cmp_branch);
16137 %}
16138 
16139 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16140   match(If cmp (CmpU op1 op2));
16141   effect(USE labl);
16142 
16143   ins_cost(BRANCH_COST);
16144   format %{ "cbw$cmp   $op1, $labl" %}
16145   ins_encode %{
16146     Label* L = $labl$$label;
16147     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16148     if (cond == Assembler::EQ || cond == Assembler::LS)
16149       __ cbzw($op1$$Register, *L);
16150     else
16151       __ cbnzw($op1$$Register, *L);
16152   %}
16153   ins_pipe(pipe_cmp_branch);
16154 %}
16155 
16156 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16157   match(If cmp (CmpUL op1 op2));
16158   effect(USE labl);
16159 
16160   ins_cost(BRANCH_COST);
16161   format %{ "cb$cmp   $op1, $labl" %}
16162   ins_encode %{
16163     Label* L = $labl$$label;
16164     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16165     if (cond == Assembler::EQ || cond == Assembler::LS)
16166       __ cbz($op1$$Register, *L);
16167     else
16168       __ cbnz($op1$$Register, *L);
16169   %}
16170   ins_pipe(pipe_cmp_branch);
16171 %}
16172 
16173 // Test bit and Branch
16174 
16175 // Patterns for short (< 32KiB) variants
16176 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16177   match(If cmp (CmpL op1 op2));
16178   effect(USE labl);
16179 
16180   ins_cost(BRANCH_COST);
16181   format %{ "cb$cmp   $op1, $labl # long" %}
16182   ins_encode %{
16183     Label* L = $labl$$label;
16184     Assembler::Condition cond =
16185       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16186     __ tbr(cond, $op1$$Register, 63, *L);
16187   %}
16188   ins_pipe(pipe_cmp_branch);
16189   ins_short_branch(1);
16190 %}
16191 
16192 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16193   match(If cmp (CmpI op1 op2));
16194   effect(USE labl);
16195 
16196   ins_cost(BRANCH_COST);
16197   format %{ "cb$cmp   $op1, $labl # int" %}
16198   ins_encode %{
16199     Label* L = $labl$$label;
16200     Assembler::Condition cond =
16201       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16202     __ tbr(cond, $op1$$Register, 31, *L);
16203   %}
16204   ins_pipe(pipe_cmp_branch);
16205   ins_short_branch(1);
16206 %}
16207 
16208 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16209   match(If cmp (CmpL (AndL op1 op2) op3));
16210   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16211   effect(USE labl);
16212 
16213   ins_cost(BRANCH_COST);
16214   format %{ "tb$cmp   $op1, $op2, $labl" %}
16215   ins_encode %{
16216     Label* L = $labl$$label;
16217     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16218     int bit = exact_log2_long($op2$$constant);
16219     __ tbr(cond, $op1$$Register, bit, *L);
16220   %}
16221   ins_pipe(pipe_cmp_branch);
16222   ins_short_branch(1);
16223 %}
16224 
16225 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16226   match(If cmp (CmpI (AndI op1 op2) op3));
16227   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16228   effect(USE labl);
16229 
16230   ins_cost(BRANCH_COST);
16231   format %{ "tb$cmp   $op1, $op2, $labl" %}
16232   ins_encode %{
16233     Label* L = $labl$$label;
16234     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16235     int bit = exact_log2((juint)$op2$$constant);
16236     __ tbr(cond, $op1$$Register, bit, *L);
16237   %}
16238   ins_pipe(pipe_cmp_branch);
16239   ins_short_branch(1);
16240 %}
16241 
16242 // And far variants
16243 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16244   match(If cmp (CmpL op1 op2));
16245   effect(USE labl);
16246 
16247   ins_cost(BRANCH_COST);
16248   format %{ "cb$cmp   $op1, $labl # long" %}
16249   ins_encode %{
16250     Label* L = $labl$$label;
16251     Assembler::Condition cond =
16252       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16253     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16254   %}
16255   ins_pipe(pipe_cmp_branch);
16256 %}
16257 
16258 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16259   match(If cmp (CmpI op1 op2));
16260   effect(USE labl);
16261 
16262   ins_cost(BRANCH_COST);
16263   format %{ "cb$cmp   $op1, $labl # int" %}
16264   ins_encode %{
16265     Label* L = $labl$$label;
16266     Assembler::Condition cond =
16267       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16268     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16269   %}
16270   ins_pipe(pipe_cmp_branch);
16271 %}
16272 
16273 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16274   match(If cmp (CmpL (AndL op1 op2) op3));
16275   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16276   effect(USE labl);
16277 
16278   ins_cost(BRANCH_COST);
16279   format %{ "tb$cmp   $op1, $op2, $labl" %}
16280   ins_encode %{
16281     Label* L = $labl$$label;
16282     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16283     int bit = exact_log2_long($op2$$constant);
16284     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16285   %}
16286   ins_pipe(pipe_cmp_branch);
16287 %}
16288 
16289 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16290   match(If cmp (CmpI (AndI op1 op2) op3));
16291   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16292   effect(USE labl);
16293 
16294   ins_cost(BRANCH_COST);
16295   format %{ "tb$cmp   $op1, $op2, $labl" %}
16296   ins_encode %{
16297     Label* L = $labl$$label;
16298     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16299     int bit = exact_log2((juint)$op2$$constant);
16300     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16301   %}
16302   ins_pipe(pipe_cmp_branch);
16303 %}
16304 
16305 // Test bits
16306 
16307 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16308   match(Set cr (CmpL (AndL op1 op2) op3));
16309   predicate(Assembler::operand_valid_for_logical_immediate
16310             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16311 
16312   ins_cost(INSN_COST);
16313   format %{ "tst $op1, $op2 # long" %}
16314   ins_encode %{
16315     __ tst($op1$$Register, $op2$$constant);
16316   %}
16317   ins_pipe(ialu_reg_reg);
16318 %}
16319 
16320 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16321   match(Set cr (CmpI (AndI op1 op2) op3));
16322   predicate(Assembler::operand_valid_for_logical_immediate
16323             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16324 
16325   ins_cost(INSN_COST);
16326   format %{ "tst $op1, $op2 # int" %}
16327   ins_encode %{
16328     __ tstw($op1$$Register, $op2$$constant);
16329   %}
16330   ins_pipe(ialu_reg_reg);
16331 %}
16332 
16333 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16334   match(Set cr (CmpL (AndL op1 op2) op3));
16335 
16336   ins_cost(INSN_COST);
16337   format %{ "tst $op1, $op2 # long" %}
16338   ins_encode %{
16339     __ tst($op1$$Register, $op2$$Register);
16340   %}
16341   ins_pipe(ialu_reg_reg);
16342 %}
16343 
16344 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16345   match(Set cr (CmpI (AndI op1 op2) op3));
16346 
16347   ins_cost(INSN_COST);
16348   format %{ "tstw $op1, $op2 # int" %}
16349   ins_encode %{
16350     __ tstw($op1$$Register, $op2$$Register);
16351   %}
16352   ins_pipe(ialu_reg_reg);
16353 %}
16354 
16355 
16356 // Conditional Far Branch
16357 // Conditional Far Branch Unsigned
16358 // TODO: fixme
16359 
16360 // counted loop end branch near
16361 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16362 %{
16363   match(CountedLoopEnd cmp cr);
16364 
16365   effect(USE lbl);
16366 
16367   ins_cost(BRANCH_COST);
16368   // short variant.
16369   // ins_short_branch(1);
16370   format %{ "b$cmp $lbl \t// counted loop end" %}
16371 
16372   ins_encode(aarch64_enc_br_con(cmp, lbl));
16373 
16374   ins_pipe(pipe_branch);
16375 %}
16376 
16377 // counted loop end branch near Unsigned
16378 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16379 %{
16380   match(CountedLoopEnd cmp cr);
16381 
16382   effect(USE lbl);
16383 
16384   ins_cost(BRANCH_COST);
16385   // short variant.
16386   // ins_short_branch(1);
16387   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16388 
16389   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16390 
16391   ins_pipe(pipe_branch);
16392 %}
16393 
16394 // counted loop end branch far
16395 // counted loop end branch far unsigned
16396 // TODO: fixme
16397 
16398 // ============================================================================
16399 // inlined locking and unlocking
16400 
16401 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16402 %{
16403   match(Set cr (FastLock object box));
16404   effect(TEMP tmp, TEMP tmp2);
16405 
16406   // TODO
16407   // identify correct cost
16408   ins_cost(5 * INSN_COST);
16409   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16410 
16411   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16412 
16413   ins_pipe(pipe_serial);
16414 %}
16415 
16416 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16417 %{
16418   match(Set cr (FastUnlock object box));
16419   effect(TEMP tmp, TEMP tmp2);
16420 
16421   ins_cost(5 * INSN_COST);
16422   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16423 
16424   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16425 
16426   ins_pipe(pipe_serial);
16427 %}
16428 
16429 
16430 // ============================================================================
16431 // Safepoint Instructions
16432 
16433 // TODO
16434 // provide a near and far version of this code
16435 
16436 instruct safePoint(rFlagsReg cr, iRegP poll)
16437 %{
16438   match(SafePoint poll);
16439   effect(KILL cr);
16440 
16441   format %{
16442     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16443   %}
16444   ins_encode %{
16445     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16446   %}
16447   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16448 %}
16449 
16450 
16451 // ============================================================================
16452 // Procedure Call/Return Instructions
16453 
16454 // Call Java Static Instruction
16455 
16456 instruct CallStaticJavaDirect(method meth)
16457 %{
16458   match(CallStaticJava);
16459 
16460   effect(USE meth);
16461 
16462   ins_cost(CALL_COST);
16463 
16464   format %{ "call,static $meth \t// ==> " %}
16465 
16466   ins_encode(aarch64_enc_java_static_call(meth),
16467              aarch64_enc_call_epilog);
16468 
16469   ins_pipe(pipe_class_call);
16470 %}
16471 
16472 // TO HERE
16473 
16474 // Call Java Dynamic Instruction
16475 instruct CallDynamicJavaDirect(method meth)
16476 %{
16477   match(CallDynamicJava);
16478 
16479   effect(USE meth);
16480 
16481   ins_cost(CALL_COST);
16482 
16483   format %{ "CALL,dynamic $meth \t// ==> " %}
16484 
16485   ins_encode(aarch64_enc_java_dynamic_call(meth),
16486              aarch64_enc_call_epilog);
16487 
16488   ins_pipe(pipe_class_call);
16489 %}
16490 
16491 // Call Runtime Instruction
16492 
16493 instruct CallRuntimeDirect(method meth)
16494 %{
16495   match(CallRuntime);
16496 
16497   effect(USE meth);
16498 
16499   ins_cost(CALL_COST);
16500 
16501   format %{ "CALL, runtime $meth" %}
16502 
16503   ins_encode( aarch64_enc_java_to_runtime(meth) );
16504 
16505   ins_pipe(pipe_class_call);
16506 %}
16507 
16508 // Call Runtime Instruction
16509 
16510 instruct CallLeafDirect(method meth)
16511 %{
16512   match(CallLeaf);
16513 
16514   effect(USE meth);
16515 
16516   ins_cost(CALL_COST);
16517 
16518   format %{ "CALL, runtime leaf $meth" %}
16519 
16520   ins_encode( aarch64_enc_java_to_runtime(meth) );
16521 
16522   ins_pipe(pipe_class_call);
16523 %}
16524 
16525 // Call Runtime Instruction
16526 
16527 instruct CallLeafNoFPDirect(method meth)
16528 %{
16529   match(CallLeafNoFP);
16530 
16531   effect(USE meth);
16532 
16533   ins_cost(CALL_COST);
16534 
16535   format %{ "CALL, runtime leaf nofp $meth" %}
16536 
16537   ins_encode( aarch64_enc_java_to_runtime(meth) );
16538 
16539   ins_pipe(pipe_class_call);
16540 %}
16541 
16542 instruct CallNativeDirect(method meth)
16543 %{
16544   match(CallNative);
16545 
16546   effect(USE meth);
16547 
16548   ins_cost(CALL_COST);
16549 
16550   format %{ "CALL, native $meth" %}
16551 
16552   ins_encode( aarch64_enc_java_to_runtime(meth) );
16553 
16554   ins_pipe(pipe_class_call);
16555 %}
16556 
16557 // Tail Call; Jump from runtime stub to Java code.
16558 // Also known as an 'interprocedural jump'.
16559 // Target of jump will eventually return to caller.
16560 // TailJump below removes the return address.
16561 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16562 %{
16563   match(TailCall jump_target method_ptr);
16564 
16565   ins_cost(CALL_COST);
16566 
16567   format %{ "br $jump_target\t# $method_ptr holds method" %}
16568 
16569   ins_encode(aarch64_enc_tail_call(jump_target));
16570 
16571   ins_pipe(pipe_class_call);
16572 %}
16573 
16574 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16575 %{
16576   match(TailJump jump_target ex_oop);
16577 
16578   ins_cost(CALL_COST);
16579 
16580   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16581 
16582   ins_encode(aarch64_enc_tail_jmp(jump_target));
16583 
16584   ins_pipe(pipe_class_call);
16585 %}
16586 
16587 // Create exception oop: created by stack-crawling runtime code.
16588 // Created exception is now available to this handler, and is setup
16589 // just prior to jumping to this handler. No code emitted.
16590 // TODO check
16591 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16592 instruct CreateException(iRegP_R0 ex_oop)
16593 %{
16594   match(Set ex_oop (CreateEx));
16595 
16596   format %{ " -- \t// exception oop; no code emitted" %}
16597 
16598   size(0);
16599 
16600   ins_encode( /*empty*/ );
16601 
16602   ins_pipe(pipe_class_empty);
16603 %}
16604 
16605 // Rethrow exception: The exception oop will come in the first
16606 // argument position. Then JUMP (not call) to the rethrow stub code.
16607 instruct RethrowException() %{
16608   match(Rethrow);
16609   ins_cost(CALL_COST);
16610 
16611   format %{ "b rethrow_stub" %}
16612 
16613   ins_encode( aarch64_enc_rethrow() );
16614 
16615   ins_pipe(pipe_class_call);
16616 %}
16617 
16618 
16619 // Return Instruction
16620 // epilog node loads ret address into lr as part of frame pop
16621 instruct Ret()
16622 %{
16623   match(Return);
16624 
16625   format %{ "ret\t// return register" %}
16626 
16627   ins_encode( aarch64_enc_ret() );
16628 
16629   ins_pipe(pipe_branch);
16630 %}
16631 
16632 // Die now.
16633 instruct ShouldNotReachHere() %{
16634   match(Halt);
16635 
16636   ins_cost(CALL_COST);
16637   format %{ "ShouldNotReachHere" %}
16638 
16639   ins_encode %{
16640     if (is_reachable()) {
16641       __ stop(_halt_reason);
16642     }
16643   %}
16644 
16645   ins_pipe(pipe_class_default);
16646 %}
16647 
16648 // ============================================================================
16649 // Partial Subtype Check
16650 //
16651 // superklass array for an instance of the superklass.  Set a hidden
16652 // internal cache on a hit (cache is checked with exposed code in
16653 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16654 // encoding ALSO sets flags.
16655 
16656 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16657 %{
16658   match(Set result (PartialSubtypeCheck sub super));
16659   effect(KILL cr, KILL temp);
16660 
16661   ins_cost(1100);  // slightly larger than the next version
16662   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16663 
16664   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16665 
16666   opcode(0x1); // Force zero of result reg on hit
16667 
16668   ins_pipe(pipe_class_memory);
16669 %}
16670 
16671 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16672 %{
16673   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16674   effect(KILL temp, KILL result);
16675 
16676   ins_cost(1100);  // slightly larger than the next version
16677   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16678 
16679   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16680 
16681   opcode(0x0); // Don't zero result reg on hit
16682 
16683   ins_pipe(pipe_class_memory);
16684 %}
16685 
16686 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16687                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16688 %{
16689   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16690   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16691   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16692 
16693   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16694   ins_encode %{
16695     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16696     __ string_compare($str1$$Register, $str2$$Register,
16697                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16698                       $tmp1$$Register, $tmp2$$Register,
16699                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16700   %}
16701   ins_pipe(pipe_class_memory);
16702 %}
16703 
16704 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16705                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16706 %{
16707   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16708   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16709   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16710 
16711   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16712   ins_encode %{
16713     __ string_compare($str1$$Register, $str2$$Register,
16714                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16715                       $tmp1$$Register, $tmp2$$Register,
16716                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16717   %}
16718   ins_pipe(pipe_class_memory);
16719 %}
16720 
16721 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16722                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16723                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16724 %{
16725   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16726   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16727   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16728          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16729 
16730   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16731   ins_encode %{
16732     __ string_compare($str1$$Register, $str2$$Register,
16733                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16734                       $tmp1$$Register, $tmp2$$Register,
16735                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16736                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16737   %}
16738   ins_pipe(pipe_class_memory);
16739 %}
16740 
16741 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16742                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16743                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16744 %{
16745   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16746   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16747   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16748          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16749 
16750   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16751   ins_encode %{
16752     __ string_compare($str1$$Register, $str2$$Register,
16753                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16754                       $tmp1$$Register, $tmp2$$Register,
16755                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16756                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16757   %}
16758   ins_pipe(pipe_class_memory);
16759 %}
16760 
16761 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16762        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16763        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16764 %{
16765   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16766   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16767   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16768          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16769   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16770 
16771   ins_encode %{
16772     __ string_indexof($str1$$Register, $str2$$Register,
16773                       $cnt1$$Register, $cnt2$$Register,
16774                       $tmp1$$Register, $tmp2$$Register,
16775                       $tmp3$$Register, $tmp4$$Register,
16776                       $tmp5$$Register, $tmp6$$Register,
16777                       -1, $result$$Register, StrIntrinsicNode::UU);
16778   %}
16779   ins_pipe(pipe_class_memory);
16780 %}
16781 
16782 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16783        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16784        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16785 %{
16786   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16787   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16788   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16789          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16790   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16791 
16792   ins_encode %{
16793     __ string_indexof($str1$$Register, $str2$$Register,
16794                       $cnt1$$Register, $cnt2$$Register,
16795                       $tmp1$$Register, $tmp2$$Register,
16796                       $tmp3$$Register, $tmp4$$Register,
16797                       $tmp5$$Register, $tmp6$$Register,
16798                       -1, $result$$Register, StrIntrinsicNode::LL);
16799   %}
16800   ins_pipe(pipe_class_memory);
16801 %}
16802 
16803 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16804        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16805        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16806 %{
16807   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16808   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16809   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16810          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16811   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16812 
16813   ins_encode %{
16814     __ string_indexof($str1$$Register, $str2$$Register,
16815                       $cnt1$$Register, $cnt2$$Register,
16816                       $tmp1$$Register, $tmp2$$Register,
16817                       $tmp3$$Register, $tmp4$$Register,
16818                       $tmp5$$Register, $tmp6$$Register,
16819                       -1, $result$$Register, StrIntrinsicNode::UL);
16820   %}
16821   ins_pipe(pipe_class_memory);
16822 %}
16823 
16824 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16825                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16826                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16827 %{
16828   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16829   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16830   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16831          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16832   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16833 
16834   ins_encode %{
16835     int icnt2 = (int)$int_cnt2$$constant;
16836     __ string_indexof($str1$$Register, $str2$$Register,
16837                       $cnt1$$Register, zr,
16838                       $tmp1$$Register, $tmp2$$Register,
16839                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16840                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16841   %}
16842   ins_pipe(pipe_class_memory);
16843 %}
16844 
16845 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16846                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16847                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16848 %{
16849   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16850   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16851   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16852          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16853   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16854 
16855   ins_encode %{
16856     int icnt2 = (int)$int_cnt2$$constant;
16857     __ string_indexof($str1$$Register, $str2$$Register,
16858                       $cnt1$$Register, zr,
16859                       $tmp1$$Register, $tmp2$$Register,
16860                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16861                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16862   %}
16863   ins_pipe(pipe_class_memory);
16864 %}
16865 
16866 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16867                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16868                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16869 %{
16870   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16871   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16872   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16873          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16874   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16875 
16876   ins_encode %{
16877     int icnt2 = (int)$int_cnt2$$constant;
16878     __ string_indexof($str1$$Register, $str2$$Register,
16879                       $cnt1$$Register, zr,
16880                       $tmp1$$Register, $tmp2$$Register,
16881                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16882                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16883   %}
16884   ins_pipe(pipe_class_memory);
16885 %}
16886 
16887 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16888                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16889                              iRegINoSp tmp3, rFlagsReg cr)
16890 %{
16891   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16892   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16893   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16894          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16895 
16896   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16897 
16898   ins_encode %{
16899     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16900                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16901                            $tmp3$$Register);
16902   %}
16903   ins_pipe(pipe_class_memory);
16904 %}
16905 
16906 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16907                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16908                               iRegINoSp tmp3, rFlagsReg cr)
16909 %{
16910   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16911   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16912   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16913          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16914 
16915   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16916 
16917   ins_encode %{
16918     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16919                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16920                             $tmp3$$Register);
16921   %}
16922   ins_pipe(pipe_class_memory);
16923 %}
16924 
16925 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16926                         iRegI_R0 result, rFlagsReg cr)
16927 %{
16928   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16929   match(Set result (StrEquals (Binary str1 str2) cnt));
16930   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16931 
16932   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16933   ins_encode %{
16934     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16935     __ string_equals($str1$$Register, $str2$$Register,
16936                      $result$$Register, $cnt$$Register, 1);
16937   %}
16938   ins_pipe(pipe_class_memory);
16939 %}
16940 
16941 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16942                         iRegI_R0 result, rFlagsReg cr)
16943 %{
16944   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16945   match(Set result (StrEquals (Binary str1 str2) cnt));
16946   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16947 
16948   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16949   ins_encode %{
16950     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16951     __ string_equals($str1$$Register, $str2$$Register,
16952                      $result$$Register, $cnt$$Register, 2);
16953   %}
16954   ins_pipe(pipe_class_memory);
16955 %}
16956 
16957 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16958                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16959                        iRegP_R10 tmp, rFlagsReg cr)
16960 %{
16961   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16962   match(Set result (AryEq ary1 ary2));
16963   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16964 
16965   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16966   ins_encode %{
16967     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16968                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16969                                    $result$$Register, $tmp$$Register, 1);
16970     if (tpc == NULL) {
16971       ciEnv::current()->record_failure("CodeCache is full");
16972       return;
16973     }
16974   %}
16975   ins_pipe(pipe_class_memory);
16976 %}
16977 
16978 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16979                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16980                        iRegP_R10 tmp, rFlagsReg cr)
16981 %{
16982   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16983   match(Set result (AryEq ary1 ary2));
16984   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16985 
16986   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16987   ins_encode %{
16988     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16989                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16990                                    $result$$Register, $tmp$$Register, 2);
16991     if (tpc == NULL) {
16992       ciEnv::current()->record_failure("CodeCache is full");
16993       return;
16994     }
16995   %}
16996   ins_pipe(pipe_class_memory);
16997 %}
16998 
16999 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17000 %{
17001   match(Set result (HasNegatives ary1 len));
17002   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17003   format %{ "has negatives byte[] $ary1,$len -> $result" %}
17004   ins_encode %{
17005     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17006     if (tpc == NULL) {
17007       ciEnv::current()->record_failure("CodeCache is full");
17008       return;
17009     }
17010   %}
17011   ins_pipe( pipe_slow );
17012 %}
17013 
17014 // fast char[] to byte[] compression
17015 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17016                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17017                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17018                          iRegI_R0 result, rFlagsReg cr)
17019 %{
17020   match(Set result (StrCompressedCopy src (Binary dst len)));
17021   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17022 
17023   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
17024   ins_encode %{
17025     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17026                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17027                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17028                            $result$$Register);
17029   %}
17030   ins_pipe( pipe_slow );
17031 %}
17032 
17033 // fast byte[] to char[] inflation
17034 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17035                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17036 %{
17037   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17038   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17039 
17040   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17041   ins_encode %{
17042     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17043                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17044                                         $tmp3$$FloatRegister, $tmp4$$Register);
17045     if (tpc == NULL) {
17046       ciEnv::current()->record_failure("CodeCache is full");
17047       return;
17048     }
17049   %}
17050   ins_pipe(pipe_class_memory);
17051 %}
17052 
17053 // encode char[] to byte[] in ISO_8859_1
17054 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17055                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17056                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17057                           iRegI_R0 result, rFlagsReg cr)
17058 %{
17059   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17060   match(Set result (EncodeISOArray src (Binary dst len)));
17061   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17062          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17063 
17064   format %{ "Encode array $src,$dst,$len -> $result" %}
17065   ins_encode %{
17066     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17067          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
17068          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
17069   %}
17070   ins_pipe( pipe_class_memory );
17071 %}
17072 
17073 // ============================================================================
17074 // This name is KNOWN by the ADLC and cannot be changed.
17075 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17076 // for this guy.
17077 instruct tlsLoadP(thread_RegP dst)
17078 %{
17079   match(Set dst (ThreadLocal));
17080 
17081   ins_cost(0);
17082 
17083   format %{ " -- \t// $dst=Thread::current(), empty" %}
17084 
17085   size(0);
17086 
17087   ins_encode( /*empty*/ );
17088 
17089   ins_pipe(pipe_class_empty);
17090 %}
17091 
17092 //----------PEEPHOLE RULES-----------------------------------------------------
17093 // These must follow all instruction definitions as they use the names
17094 // defined in the instructions definitions.
17095 //
17096 // peepmatch ( root_instr_name [preceding_instruction]* );
17097 //
17098 // peepconstraint %{
17099 // (instruction_number.operand_name relational_op instruction_number.operand_name
17100 //  [, ...] );
17101 // // instruction numbers are zero-based using left to right order in peepmatch
17102 //
17103 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17104 // // provide an instruction_number.operand_name for each operand that appears
17105 // // in the replacement instruction's match rule
17106 //
17107 // ---------VM FLAGS---------------------------------------------------------
17108 //
17109 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17110 //
17111 // Each peephole rule is given an identifying number starting with zero and
17112 // increasing by one in the order seen by the parser.  An individual peephole
17113 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17114 // on the command-line.
17115 //
17116 // ---------CURRENT LIMITATIONS----------------------------------------------
17117 //
17118 // Only match adjacent instructions in same basic block
17119 // Only equality constraints
17120 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17121 // Only one replacement instruction
17122 //
17123 // ---------EXAMPLE----------------------------------------------------------
17124 //
17125 // // pertinent parts of existing instructions in architecture description
17126 // instruct movI(iRegINoSp dst, iRegI src)
17127 // %{
17128 //   match(Set dst (CopyI src));
17129 // %}
17130 //
17131 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17132 // %{
17133 //   match(Set dst (AddI dst src));
17134 //   effect(KILL cr);
17135 // %}
17136 //
17137 // // Change (inc mov) to lea
17138 // peephole %{
17139 //   // increment preceeded by register-register move
17140 //   peepmatch ( incI_iReg movI );
17141 //   // require that the destination register of the increment
17142 //   // match the destination register of the move
17143 //   peepconstraint ( 0.dst == 1.dst );
17144 //   // construct a replacement instruction that sets
17145 //   // the destination to ( move's source register + one )
17146 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17147 // %}
17148 //
17149 
17150 // Implementation no longer uses movX instructions since
17151 // machine-independent system no longer uses CopyX nodes.
17152 //
17153 // peephole
17154 // %{
17155 //   peepmatch (incI_iReg movI);
17156 //   peepconstraint (0.dst == 1.dst);
17157 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17158 // %}
17159 
17160 // peephole
17161 // %{
17162 //   peepmatch (decI_iReg movI);
17163 //   peepconstraint (0.dst == 1.dst);
17164 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17165 // %}
17166 
17167 // peephole
17168 // %{
17169 //   peepmatch (addI_iReg_imm movI);
17170 //   peepconstraint (0.dst == 1.dst);
17171 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17172 // %}
17173 
17174 // peephole
17175 // %{
17176 //   peepmatch (incL_iReg movL);
17177 //   peepconstraint (0.dst == 1.dst);
17178 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17179 // %}
17180 
17181 // peephole
17182 // %{
17183 //   peepmatch (decL_iReg movL);
17184 //   peepconstraint (0.dst == 1.dst);
17185 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17186 // %}
17187 
17188 // peephole
17189 // %{
17190 //   peepmatch (addL_iReg_imm movL);
17191 //   peepconstraint (0.dst == 1.dst);
17192 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17193 // %}
17194 
17195 // peephole
17196 // %{
17197 //   peepmatch (addP_iReg_imm movP);
17198 //   peepconstraint (0.dst == 1.dst);
17199 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17200 // %}
17201 
17202 // // Change load of spilled value to only a spill
17203 // instruct storeI(memory mem, iRegI src)
17204 // %{
17205 //   match(Set mem (StoreI mem src));
17206 // %}
17207 //
17208 // instruct loadI(iRegINoSp dst, memory mem)
17209 // %{
17210 //   match(Set dst (LoadI mem));
17211 // %}
17212 //
17213 
17214 //----------SMARTSPILL RULES---------------------------------------------------
17215 // These must follow all instruction definitions as they use the names
17216 // defined in the instructions definitions.
17217 
17218 // Local Variables:
17219 // mode: c++
17220 // End:
--- EOF ---