1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298  bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() >= 16) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   int max_monitors = C->method() != NULL ? C->max_monitors() : 0;
 1916   if (UseFastLocking && max_monitors > 0) {
 1917     C2CheckLockStackStub* stub = new (C->comp_arena()) C2CheckLockStackStub();
 1918     C->output()->add_stub(stub);
 1919     __ ldr(r9, Address(rthread, JavaThread::lock_stack_current_offset()));
 1920     __ add(r9, r9, max_monitors * oopSize);
 1921     __ ldr(r10, Address(rthread, JavaThread::lock_stack_limit_offset()));
 1922     __ cmp(r9, r10);
 1923     __ br(Assembler::GE, stub->entry());
 1924     __ bind(stub->continuation());
 1925   }
 1926 
 1927   if (C->stub_function() == NULL) {
 1928     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1929     bs->nmethod_entry_barrier(&_masm);
 1930   }
 1931 
 1932   if (VerifyStackAtCalls) {
 1933     Unimplemented();
 1934   }
 1935 
 1936   C->output()->set_frame_complete(cbuf.insts_size());
 1937 
 1938   if (C->has_mach_constant_base_node()) {
 1939     // NOTE: We set the table base offset here because users might be
 1940     // emitted before MachConstantBaseNode.
 1941     ConstantTable& constant_table = C->output()->constant_table();
 1942     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1943   }
 1944 }
 1945 
 1946 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1947 {
 1948   return MachNode::size(ra_); // too many variables; just compute it
 1949                               // the hard way
 1950 }
 1951 
 1952 int MachPrologNode::reloc() const
 1953 {
 1954   return 0;
 1955 }
 1956 
 1957 //=============================================================================
 1958 
 1959 #ifndef PRODUCT
 1960 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1961   Compile* C = ra_->C;
 1962   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1963 
 1964   st->print("# pop frame %d\n\t",framesize);
 1965 
 1966   if (framesize == 0) {
 1967     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1968   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1969     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1970     st->print("add  sp, sp, #%d\n\t", framesize);
 1971   } else {
 1972     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1973     st->print("add  sp, sp, rscratch1\n\t");
 1974     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1975   }
 1976 
 1977   if (do_polling() && C->is_method_compilation()) {
 1978     st->print("# test polling word\n\t");
 1979     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1980     st->print("cmp  sp, rscratch1\n\t");
 1981     st->print("bhi #slow_path");
 1982   }
 1983 }
 1984 #endif
 1985 
 1986 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1987   Compile* C = ra_->C;
 1988   C2_MacroAssembler _masm(&cbuf);
 1989   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1990 
 1991   __ remove_frame(framesize);
 1992 
 1993   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1994     __ reserved_stack_check();
 1995   }
 1996 
 1997   if (do_polling() && C->is_method_compilation()) {
 1998     Label dummy_label;
 1999     Label* code_stub = &dummy_label;
 2000     if (!C->output()->in_scratch_emit_size()) {
 2001       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 2002       C->output()->add_stub(stub);
 2003       code_stub = &stub->entry();
 2004     }
 2005     __ relocate(relocInfo::poll_return_type);
 2006     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 2007   }
 2008 }
 2009 
 2010 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2011   // Variable size. Determine dynamically.
 2012   return MachNode::size(ra_);
 2013 }
 2014 
 2015 int MachEpilogNode::reloc() const {
 2016   // Return number of relocatable values contained in this instruction.
 2017   return 1; // 1 for polling page.
 2018 }
 2019 
 2020 const Pipeline * MachEpilogNode::pipeline() const {
 2021   return MachNode::pipeline_class();
 2022 }
 2023 
 2024 //=============================================================================
 2025 
 2026 // Figure out which register class each belongs in: rc_int, rc_float or
 2027 // rc_stack.
 2028 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2029 
 2030 static enum RC rc_class(OptoReg::Name reg) {
 2031 
 2032   if (reg == OptoReg::Bad) {
 2033     return rc_bad;
 2034   }
 2035 
 2036   // we have 32 int registers * 2 halves
 2037   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2038 
 2039   if (reg < slots_of_int_registers) {
 2040     return rc_int;
 2041   }
 2042 
 2043   // we have 32 float register * 8 halves
 2044   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2045   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2046     return rc_float;
 2047   }
 2048 
 2049   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2050   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2051     return rc_predicate;
 2052   }
 2053 
 2054   // Between predicate regs & stack is the flags.
 2055   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2056 
 2057   return rc_stack;
 2058 }
 2059 
 2060 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2061   Compile* C = ra_->C;
 2062 
 2063   // Get registers to move.
 2064   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2065   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2066   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2067   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2068 
 2069   enum RC src_hi_rc = rc_class(src_hi);
 2070   enum RC src_lo_rc = rc_class(src_lo);
 2071   enum RC dst_hi_rc = rc_class(dst_hi);
 2072   enum RC dst_lo_rc = rc_class(dst_lo);
 2073 
 2074   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2075 
 2076   if (src_hi != OptoReg::Bad) {
 2077     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2078            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2079            "expected aligned-adjacent pairs");
 2080   }
 2081 
 2082   if (src_lo == dst_lo && src_hi == dst_hi) {
 2083     return 0;            // Self copy, no move.
 2084   }
 2085 
 2086   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2087               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2088   int src_offset = ra_->reg2offset(src_lo);
 2089   int dst_offset = ra_->reg2offset(dst_lo);
 2090 
 2091   if (bottom_type()->isa_vect() != NULL) {
 2092     uint ireg = ideal_reg();
 2093     if (ireg == Op_VecA && cbuf) {
 2094       C2_MacroAssembler _masm(cbuf);
 2095       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2096       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2097         // stack->stack
 2098         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2099                                                 sve_vector_reg_size_in_bytes);
 2100       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2101         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2102                             sve_vector_reg_size_in_bytes);
 2103       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2104         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2105                               sve_vector_reg_size_in_bytes);
 2106       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2107         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2108                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2109                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2110       } else {
 2111         ShouldNotReachHere();
 2112       }
 2113     } else if (cbuf) {
 2114       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2115       C2_MacroAssembler _masm(cbuf);
 2116       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2117       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2118         // stack->stack
 2119         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2120         if (ireg == Op_VecD) {
 2121           __ unspill(rscratch1, true, src_offset);
 2122           __ spill(rscratch1, true, dst_offset);
 2123         } else {
 2124           __ spill_copy128(src_offset, dst_offset);
 2125         }
 2126       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2127         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2128                ireg == Op_VecD ? __ T8B : __ T16B,
 2129                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2130       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2131         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2132                  ireg == Op_VecD ? __ D : __ Q,
 2133                  ra_->reg2offset(dst_lo));
 2134       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2135         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2136                    ireg == Op_VecD ? __ D : __ Q,
 2137                    ra_->reg2offset(src_lo));
 2138       } else {
 2139         ShouldNotReachHere();
 2140       }
 2141     }
 2142   } else if (cbuf) {
 2143     C2_MacroAssembler _masm(cbuf);
 2144     switch (src_lo_rc) {
 2145     case rc_int:
 2146       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2147         if (is64) {
 2148             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2149                    as_Register(Matcher::_regEncode[src_lo]));
 2150         } else {
 2151             C2_MacroAssembler _masm(cbuf);
 2152             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2153                     as_Register(Matcher::_regEncode[src_lo]));
 2154         }
 2155       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2156         if (is64) {
 2157             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2158                      as_Register(Matcher::_regEncode[src_lo]));
 2159         } else {
 2160             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2161                      as_Register(Matcher::_regEncode[src_lo]));
 2162         }
 2163       } else {                    // gpr --> stack spill
 2164         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2165         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2166       }
 2167       break;
 2168     case rc_float:
 2169       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2170         if (is64) {
 2171             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2172                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2173         } else {
 2174             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2175                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2176         }
 2177       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2178           if (cbuf) {
 2179             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2180                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2181         } else {
 2182             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2183                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2184         }
 2185       } else {                    // fpr --> stack spill
 2186         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2187         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2188                  is64 ? __ D : __ S, dst_offset);
 2189       }
 2190       break;
 2191     case rc_stack:
 2192       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2193         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2194       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2195         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2196                    is64 ? __ D : __ S, src_offset);
 2197       } else {                    // stack --> stack copy
 2198         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2199         __ unspill(rscratch1, is64, src_offset);
 2200         __ spill(rscratch1, is64, dst_offset);
 2201       }
 2202       break;
 2203     default:
 2204       assert(false, "bad rc_class for spill");
 2205       ShouldNotReachHere();
 2206     }
 2207   }
 2208 
 2209   if (st) {
 2210     st->print("spill ");
 2211     if (src_lo_rc == rc_stack) {
 2212       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2213     } else {
 2214       st->print("%s -> ", Matcher::regName[src_lo]);
 2215     }
 2216     if (dst_lo_rc == rc_stack) {
 2217       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2218     } else {
 2219       st->print("%s", Matcher::regName[dst_lo]);
 2220     }
 2221     if (bottom_type()->isa_vect() != NULL) {
 2222       int vsize = 0;
 2223       switch (ideal_reg()) {
 2224       case Op_VecD:
 2225         vsize = 64;
 2226         break;
 2227       case Op_VecX:
 2228         vsize = 128;
 2229         break;
 2230       case Op_VecA:
 2231         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2232         break;
 2233       default:
 2234         assert(false, "bad register type for spill");
 2235         ShouldNotReachHere();
 2236       }
 2237       st->print("\t# vector spill size = %d", vsize);
 2238     } else {
 2239       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2240     }
 2241   }
 2242 
 2243   return 0;
 2244 
 2245 }
 2246 
 2247 #ifndef PRODUCT
 2248 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2249   if (!ra_)
 2250     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2251   else
 2252     implementation(NULL, ra_, false, st);
 2253 }
 2254 #endif
 2255 
 2256 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2257   implementation(&cbuf, ra_, false, NULL);
 2258 }
 2259 
 2260 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2261   return MachNode::size(ra_);
 2262 }
 2263 
 2264 //=============================================================================
 2265 
 2266 #ifndef PRODUCT
 2267 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2268   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2269   int reg = ra_->get_reg_first(this);
 2270   st->print("add %s, rsp, #%d]\t# box lock",
 2271             Matcher::regName[reg], offset);
 2272 }
 2273 #endif
 2274 
 2275 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2276   C2_MacroAssembler _masm(&cbuf);
 2277 
 2278   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2279   int reg    = ra_->get_encode(this);
 2280 
 2281   // This add will handle any 24-bit signed offset. 24 bits allows an
 2282   // 8 megabyte stack frame.
 2283   __ add(as_Register(reg), sp, offset);
 2284 }
 2285 
 2286 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2287   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2288   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2289 
 2290   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2291     return NativeInstruction::instruction_size;
 2292   } else {
 2293     return 2 * NativeInstruction::instruction_size;
 2294   }
 2295 }
 2296 
 2297 //=============================================================================
 2298 
 2299 #ifndef PRODUCT
 2300 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2301 {
 2302   st->print_cr("# MachUEPNode");
 2303   if (UseCompressedClassPointers) {
 2304     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2305     if (CompressedKlassPointers::shift() != 0) {
 2306       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2307     }
 2308   } else {
 2309    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2310   }
 2311   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2312   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2313 }
 2314 #endif
 2315 
 2316 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2317 {
 2318   // This is the unverified entry point.
 2319   C2_MacroAssembler _masm(&cbuf);
 2320 
 2321   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2322   Label skip;
 2323   // TODO
 2324   // can we avoid this skip and still use a reloc?
 2325   __ br(Assembler::EQ, skip);
 2326   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2327   __ bind(skip);
 2328 }
 2329 
 2330 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2331 {
 2332   return MachNode::size(ra_);
 2333 }
 2334 
 2335 // REQUIRED EMIT CODE
 2336 
 2337 //=============================================================================
 2338 
 2339 // Emit exception handler code.
 2340 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2341 {
 2342   // mov rscratch1 #exception_blob_entry_point
 2343   // br rscratch1
 2344   // Note that the code buffer's insts_mark is always relative to insts.
 2345   // That's why we must use the macroassembler to generate a handler.
 2346   C2_MacroAssembler _masm(&cbuf);
 2347   address base = __ start_a_stub(size_exception_handler());
 2348   if (base == NULL) {
 2349     ciEnv::current()->record_failure("CodeCache is full");
 2350     return 0;  // CodeBuffer::expand failed
 2351   }
 2352   int offset = __ offset();
 2353   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2354   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2355   __ end_a_stub();
 2356   return offset;
 2357 }
 2358 
 2359 // Emit deopt handler code.
 2360 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2361 {
 2362   // Note that the code buffer's insts_mark is always relative to insts.
 2363   // That's why we must use the macroassembler to generate a handler.
 2364   C2_MacroAssembler _masm(&cbuf);
 2365   address base = __ start_a_stub(size_deopt_handler());
 2366   if (base == NULL) {
 2367     ciEnv::current()->record_failure("CodeCache is full");
 2368     return 0;  // CodeBuffer::expand failed
 2369   }
 2370   int offset = __ offset();
 2371 
 2372   __ adr(lr, __ pc());
 2373   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2374 
 2375   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2376   __ end_a_stub();
 2377   return offset;
 2378 }
 2379 
 2380 // REQUIRED MATCHER CODE
 2381 
 2382 //=============================================================================
 2383 
 2384 const bool Matcher::match_rule_supported(int opcode) {
 2385   if (!has_match_rule(opcode))
 2386     return false;
 2387 
 2388   bool ret_value = true;
 2389   switch (opcode) {
 2390     case Op_OnSpinWait:
 2391       return VM_Version::supports_on_spin_wait();
 2392     case Op_CacheWB:
 2393     case Op_CacheWBPreSync:
 2394     case Op_CacheWBPostSync:
 2395       if (!VM_Version::supports_data_cache_line_flush()) {
 2396         ret_value = false;
 2397       }
 2398       break;
 2399   }
 2400 
 2401   return ret_value; // Per default match rules are supported.
 2402 }
 2403 
 2404 // Identify extra cases that we might want to provide match rules for vector nodes and
 2405 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2406 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2407   if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
 2408     return false;
 2409   }
 2410   int bit_size = vlen * type2aelembytes(bt) * 8;
 2411   if (UseSVE == 0 && bit_size > 128) {
 2412     return false;
 2413   }
 2414   if (UseSVE > 0) {
 2415     return op_sve_supported(opcode);
 2416   } else { // NEON
 2417     // Special cases
 2418     switch (opcode) {
 2419     case Op_VectorMaskCmp:
 2420     // We don't have VectorReinterpret with bit_size less than 64 support for
 2421     // now, even for byte type. To be refined with fully VectorCast support.
 2422     case Op_VectorReinterpret:
 2423       if (vlen < 2 || bit_size < 64) {
 2424         return false;
 2425       }
 2426       break;
 2427     case Op_MulAddVS2VI:
 2428       if (bit_size < 128) {
 2429         return false;
 2430       }
 2431       break;
 2432     case Op_MulVL:
 2433       return false;
 2434     case Op_VectorLoadShuffle:
 2435     case Op_VectorRearrange:
 2436       if (vlen < 4) {
 2437         return false;
 2438       }
 2439       break;
 2440     // Some types of VectorCast are not implemented for now.
 2441     case Op_VectorCastI2X:
 2442       if (bt == T_BYTE) {
 2443         return false;
 2444       }
 2445       break;
 2446     case Op_VectorCastS2X:
 2447       if (vlen < 4 || bit_size < 64) {
 2448         return false;
 2449       }
 2450       break;
 2451     case Op_VectorCastF2X:
 2452     case Op_VectorCastD2X:
 2453       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2454         return false;
 2455       }
 2456       break;
 2457     default:
 2458       break;
 2459     }
 2460   }
 2461   return true; // Per default match rules are supported.
 2462 }
 2463 
 2464 const RegMask* Matcher::predicate_reg_mask(void) {
 2465   return &_PR_REG_mask;
 2466 }
 2467 
 2468 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2469   return new TypeVectMask(elemTy, length);
 2470 }
 2471 
 2472 // Vector calling convention not yet implemented.
 2473 const bool Matcher::supports_vector_calling_convention(void) {
 2474   return false;
 2475 }
 2476 
 2477 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2478   Unimplemented();
 2479   return OptoRegPair(0, 0);
 2480 }
 2481 
 2482 const int Matcher::float_pressure(int default_pressure_threshold) {
 2483   return default_pressure_threshold;
 2484 }
 2485 
 2486 // Is this branch offset short enough that a short branch can be used?
 2487 //
 2488 // NOTE: If the platform does not provide any short branch variants, then
 2489 //       this method should return false for offset 0.
 2490 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2491   // The passed offset is relative to address of the branch.
 2492 
 2493   return (-32768 <= offset && offset < 32768);
 2494 }
 2495 
 2496 // Vector width in bytes.
 2497 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2498   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2499   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2500   // Minimum 2 values in vector
 2501   if (size < 2*type2aelembytes(bt)) size = 0;
 2502   // But never < 4
 2503   if (size < 4) size = 0;
 2504   return size;
 2505 }
 2506 
 2507 // Limits on vector size (number of elements) loaded into vector.
 2508 const int Matcher::max_vector_size(const BasicType bt) {
 2509   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2510 }
 2511 const int Matcher::min_vector_size(const BasicType bt) {
 2512   int max_size = max_vector_size(bt);
 2513   if ((UseSVE > 0) && (MaxVectorSize >= 16)) {
 2514     // Currently vector length less than SVE vector register size is not supported.
 2515     return max_size;
 2516   } else { // NEON
 2517     // Limit the vector size to 8 bytes
 2518     int size = 8 / type2aelembytes(bt);
 2519     if (bt == T_BYTE) {
 2520       // To support vector api shuffle/rearrange.
 2521       size = 4;
 2522     } else if (bt == T_BOOLEAN) {
 2523       // To support vector api load/store mask.
 2524       size = 2;
 2525     }
 2526     if (size < 2) size = 2;
 2527     return MIN2(size,max_size);
 2528   }
 2529 }
 2530 
 2531 // Actual max scalable vector register length.
 2532 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2533   return Matcher::max_vector_size(bt);
 2534 }
 2535 
 2536 // Vector ideal reg.
 2537 const uint Matcher::vector_ideal_reg(int len) {
 2538   if (UseSVE > 0 && 16 <= len && len <= 256) {
 2539     return Op_VecA;
 2540   }
 2541   switch(len) {
 2542     // For 16-bit/32-bit mask vector, reuse VecD.
 2543     case  2:
 2544     case  4:
 2545     case  8: return Op_VecD;
 2546     case 16: return Op_VecX;
 2547   }
 2548   ShouldNotReachHere();
 2549   return 0;
 2550 }
 2551 
 2552 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2553   ShouldNotReachHere(); // generic vector operands not supported
 2554   return NULL;
 2555 }
 2556 
 2557 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
 2558   ShouldNotReachHere();  // generic vector operands not supported
 2559   return false;
 2560 }
 2561 
 2562 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2563   ShouldNotReachHere();  // generic vector operands not supported
 2564   return false;
 2565 }
 2566 
 2567 // Return whether or not this register is ever used as an argument.
 2568 // This function is used on startup to build the trampoline stubs in
 2569 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2570 // call in the trampoline, and arguments in those registers not be
 2571 // available to the callee.
 2572 bool Matcher::can_be_java_arg(int reg)
 2573 {
 2574   return
 2575     reg ==  R0_num || reg == R0_H_num ||
 2576     reg ==  R1_num || reg == R1_H_num ||
 2577     reg ==  R2_num || reg == R2_H_num ||
 2578     reg ==  R3_num || reg == R3_H_num ||
 2579     reg ==  R4_num || reg == R4_H_num ||
 2580     reg ==  R5_num || reg == R5_H_num ||
 2581     reg ==  R6_num || reg == R6_H_num ||
 2582     reg ==  R7_num || reg == R7_H_num ||
 2583     reg ==  V0_num || reg == V0_H_num ||
 2584     reg ==  V1_num || reg == V1_H_num ||
 2585     reg ==  V2_num || reg == V2_H_num ||
 2586     reg ==  V3_num || reg == V3_H_num ||
 2587     reg ==  V4_num || reg == V4_H_num ||
 2588     reg ==  V5_num || reg == V5_H_num ||
 2589     reg ==  V6_num || reg == V6_H_num ||
 2590     reg ==  V7_num || reg == V7_H_num;
 2591 }
 2592 
 2593 bool Matcher::is_spillable_arg(int reg)
 2594 {
 2595   return can_be_java_arg(reg);
 2596 }
 2597 
 2598 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2599   return false;
 2600 }
 2601 
 2602 RegMask Matcher::divI_proj_mask() {
 2603   ShouldNotReachHere();
 2604   return RegMask();
 2605 }
 2606 
 2607 // Register for MODI projection of divmodI.
 2608 RegMask Matcher::modI_proj_mask() {
 2609   ShouldNotReachHere();
 2610   return RegMask();
 2611 }
 2612 
 2613 // Register for DIVL projection of divmodL.
 2614 RegMask Matcher::divL_proj_mask() {
 2615   ShouldNotReachHere();
 2616   return RegMask();
 2617 }
 2618 
 2619 // Register for MODL projection of divmodL.
 2620 RegMask Matcher::modL_proj_mask() {
 2621   ShouldNotReachHere();
 2622   return RegMask();
 2623 }
 2624 
 2625 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2626   return FP_REG_mask();
 2627 }
 2628 
 2629 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2630   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2631     Node* u = addp->fast_out(i);
 2632     if (u->is_LoadStore()) {
 2633       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2634       // instructions) only take register indirect as an operand, so
 2635       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2636       // must fail.
 2637       return false;
 2638     }
 2639     if (u->is_Mem()) {
 2640       int opsize = u->as_Mem()->memory_size();
 2641       assert(opsize > 0, "unexpected memory operand size");
 2642       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2643         return false;
 2644       }
 2645     }
 2646   }
 2647   return true;
 2648 }
 2649 
 2650 // Should the matcher clone input 'm' of node 'n'?
 2651 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2652   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2653     mstack.push(m, Visit);           // m = ShiftCntV
 2654     return true;
 2655   }
 2656   return false;
 2657 }
 2658 
 2659 // Should the Matcher clone shifts on addressing modes, expecting them
 2660 // to be subsumed into complex addressing expressions or compute them
 2661 // into registers?
 2662 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2663   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2664     return true;
 2665   }
 2666 
 2667   Node *off = m->in(AddPNode::Offset);
 2668   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2669       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2670       // Are there other uses besides address expressions?
 2671       !is_visited(off)) {
 2672     address_visited.set(off->_idx); // Flag as address_visited
 2673     mstack.push(off->in(2), Visit);
 2674     Node *conv = off->in(1);
 2675     if (conv->Opcode() == Op_ConvI2L &&
 2676         // Are there other uses besides address expressions?
 2677         !is_visited(conv)) {
 2678       address_visited.set(conv->_idx); // Flag as address_visited
 2679       mstack.push(conv->in(1), Pre_Visit);
 2680     } else {
 2681       mstack.push(conv, Pre_Visit);
 2682     }
 2683     address_visited.test_set(m->_idx); // Flag as address_visited
 2684     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2685     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2686     return true;
 2687   } else if (off->Opcode() == Op_ConvI2L &&
 2688              // Are there other uses besides address expressions?
 2689              !is_visited(off)) {
 2690     address_visited.test_set(m->_idx); // Flag as address_visited
 2691     address_visited.set(off->_idx); // Flag as address_visited
 2692     mstack.push(off->in(1), Pre_Visit);
 2693     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2694     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2695     return true;
 2696   }
 2697   return false;
 2698 }
 2699 
 2700 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2701   C2_MacroAssembler _masm(&cbuf);                                       \
 2702   {                                                                     \
 2703     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2704     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2705     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2706     __ INSN(REG, as_Register(BASE));                                    \
 2707   }
 2708 
 2709 
 2710 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2711   {
 2712     Address::extend scale;
 2713 
 2714     // Hooboy, this is fugly.  We need a way to communicate to the
 2715     // encoder that the index needs to be sign extended, so we have to
 2716     // enumerate all the cases.
 2717     switch (opcode) {
 2718     case INDINDEXSCALEDI2L:
 2719     case INDINDEXSCALEDI2LN:
 2720     case INDINDEXI2L:
 2721     case INDINDEXI2LN:
 2722       scale = Address::sxtw(size);
 2723       break;
 2724     default:
 2725       scale = Address::lsl(size);
 2726     }
 2727 
 2728     if (index == -1) {
 2729       return Address(base, disp);
 2730     } else {
 2731       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2732       return Address(base, as_Register(index), scale);
 2733     }
 2734   }
 2735 
 2736 
 2737 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2738 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2739 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2740 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2741                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2742 
 2743   // Used for all non-volatile memory accesses.  The use of
 2744   // $mem->opcode() to discover whether this pattern uses sign-extended
 2745   // offsets is something of a kludge.
 2746   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2747                         Register reg, int opcode,
 2748                         Register base, int index, int scale, int disp,
 2749                         int size_in_memory)
 2750   {
 2751     Address addr = mem2address(opcode, base, index, scale, disp);
 2752     if (addr.getMode() == Address::base_plus_offset) {
 2753       /* If we get an out-of-range offset it is a bug in the compiler,
 2754          so we assert here. */
 2755       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2756              "c2 compiler bug");
 2757       /* Fix up any out-of-range offsets. */
 2758       assert_different_registers(rscratch1, base);
 2759       assert_different_registers(rscratch1, reg);
 2760       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2761     }
 2762     (masm.*insn)(reg, addr);
 2763   }
 2764 
 2765   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2766                         FloatRegister reg, int opcode,
 2767                         Register base, int index, int size, int disp,
 2768                         int size_in_memory)
 2769   {
 2770     Address::extend scale;
 2771 
 2772     switch (opcode) {
 2773     case INDINDEXSCALEDI2L:
 2774     case INDINDEXSCALEDI2LN:
 2775       scale = Address::sxtw(size);
 2776       break;
 2777     default:
 2778       scale = Address::lsl(size);
 2779     }
 2780 
 2781     if (index == -1) {
 2782       /* If we get an out-of-range offset it is a bug in the compiler,
 2783          so we assert here. */
 2784       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2785       /* Fix up any out-of-range offsets. */
 2786       assert_different_registers(rscratch1, base);
 2787       Address addr = Address(base, disp);
 2788       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2789       (masm.*insn)(reg, addr);
 2790     } else {
 2791       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2792       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2793     }
 2794   }
 2795 
 2796   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2797                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2798                         int opcode, Register base, int index, int size, int disp)
 2799   {
 2800     if (index == -1) {
 2801       (masm.*insn)(reg, T, Address(base, disp));
 2802     } else {
 2803       assert(disp == 0, "unsupported address mode");
 2804       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2805     }
 2806   }
 2807 
 2808 %}
 2809 
 2810 
 2811 
 2812 //----------ENCODING BLOCK-----------------------------------------------------
 2813 // This block specifies the encoding classes used by the compiler to
 2814 // output byte streams.  Encoding classes are parameterized macros
 2815 // used by Machine Instruction Nodes in order to generate the bit
 2816 // encoding of the instruction.  Operands specify their base encoding
 2817 // interface with the interface keyword.  There are currently
 2818 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2819 // COND_INTER.  REG_INTER causes an operand to generate a function
 2820 // which returns its register number when queried.  CONST_INTER causes
 2821 // an operand to generate a function which returns the value of the
 2822 // constant when queried.  MEMORY_INTER causes an operand to generate
 2823 // four functions which return the Base Register, the Index Register,
 2824 // the Scale Value, and the Offset Value of the operand when queried.
 2825 // COND_INTER causes an operand to generate six functions which return
 2826 // the encoding code (ie - encoding bits for the instruction)
 2827 // associated with each basic boolean condition for a conditional
 2828 // instruction.
 2829 //
 2830 // Instructions specify two basic values for encoding.  Again, a
 2831 // function is available to check if the constant displacement is an
 2832 // oop. They use the ins_encode keyword to specify their encoding
 2833 // classes (which must be a sequence of enc_class names, and their
 2834 // parameters, specified in the encoding block), and they use the
 2835 // opcode keyword to specify, in order, their primary, secondary, and
 2836 // tertiary opcode.  Only the opcode sections which a particular
 2837 // instruction needs for encoding need to be specified.
 2838 encode %{
 2839   // Build emit functions for each basic byte or larger field in the
 2840   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2841   // from C++ code in the enc_class source block.  Emit functions will
 2842   // live in the main source block for now.  In future, we can
 2843   // generalize this by adding a syntax that specifies the sizes of
 2844   // fields in an order, so that the adlc can build the emit functions
 2845   // automagically
 2846 
 2847   // catch all for unimplemented encodings
 2848   enc_class enc_unimplemented %{
 2849     C2_MacroAssembler _masm(&cbuf);
 2850     __ unimplemented("C2 catch all");
 2851   %}
 2852 
 2853   // BEGIN Non-volatile memory access
 2854 
 2855   // This encoding class is generated automatically from ad_encode.m4.
 2856   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2857   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2858     Register dst_reg = as_Register($dst$$reg);
 2859     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2860                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2861   %}
 2862 
 2863   // This encoding class is generated automatically from ad_encode.m4.
 2864   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2865   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2866     Register dst_reg = as_Register($dst$$reg);
 2867     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2868                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2869   %}
 2870 
 2871   // This encoding class is generated automatically from ad_encode.m4.
 2872   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2873   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2874     Register dst_reg = as_Register($dst$$reg);
 2875     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2876                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2877   %}
 2878 
 2879   // This encoding class is generated automatically from ad_encode.m4.
 2880   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2881   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2882     Register dst_reg = as_Register($dst$$reg);
 2883     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2884                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2885   %}
 2886 
 2887   // This encoding class is generated automatically from ad_encode.m4.
 2888   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2889   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2890     Register dst_reg = as_Register($dst$$reg);
 2891     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2892                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2893   %}
 2894 
 2895   // This encoding class is generated automatically from ad_encode.m4.
 2896   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2897   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2898     Register dst_reg = as_Register($dst$$reg);
 2899     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2900                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2901   %}
 2902 
 2903   // This encoding class is generated automatically from ad_encode.m4.
 2904   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2905   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2906     Register dst_reg = as_Register($dst$$reg);
 2907     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2908                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2909   %}
 2910 
 2911   // This encoding class is generated automatically from ad_encode.m4.
 2912   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2913   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2914     Register dst_reg = as_Register($dst$$reg);
 2915     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2916                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2917   %}
 2918 
 2919   // This encoding class is generated automatically from ad_encode.m4.
 2920   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2921   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2922     Register dst_reg = as_Register($dst$$reg);
 2923     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2924                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2925   %}
 2926 
 2927   // This encoding class is generated automatically from ad_encode.m4.
 2928   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2929   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2930     Register dst_reg = as_Register($dst$$reg);
 2931     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2932                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2933   %}
 2934 
 2935   // This encoding class is generated automatically from ad_encode.m4.
 2936   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2937   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2938     Register dst_reg = as_Register($dst$$reg);
 2939     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2940                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2941   %}
 2942 
 2943   // This encoding class is generated automatically from ad_encode.m4.
 2944   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2945   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2946     Register dst_reg = as_Register($dst$$reg);
 2947     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2948                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2949   %}
 2950 
 2951   // This encoding class is generated automatically from ad_encode.m4.
 2952   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2953   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2954     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2955     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2956                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2957   %}
 2958 
 2959   // This encoding class is generated automatically from ad_encode.m4.
 2960   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2961   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2962     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2963     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2964                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2965   %}
 2966 
 2967   // This encoding class is generated automatically from ad_encode.m4.
 2968   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2969   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2970     Register src_reg = as_Register($src$$reg);
 2971     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2972                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2973   %}
 2974 
 2975   // This encoding class is generated automatically from ad_encode.m4.
 2976   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2977   enc_class aarch64_enc_strb0(memory1 mem) %{
 2978     C2_MacroAssembler _masm(&cbuf);
 2979     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2980                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2981   %}
 2982 
 2983   // This encoding class is generated automatically from ad_encode.m4.
 2984   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2985   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2986     Register src_reg = as_Register($src$$reg);
 2987     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2988                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2989   %}
 2990 
 2991   // This encoding class is generated automatically from ad_encode.m4.
 2992   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2993   enc_class aarch64_enc_strh0(memory2 mem) %{
 2994     C2_MacroAssembler _masm(&cbuf);
 2995     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2997   %}
 2998 
 2999   // This encoding class is generated automatically from ad_encode.m4.
 3000   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3001   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3002     Register src_reg = as_Register($src$$reg);
 3003     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3004                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3005   %}
 3006 
 3007   // This encoding class is generated automatically from ad_encode.m4.
 3008   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3009   enc_class aarch64_enc_strw0(memory4 mem) %{
 3010     C2_MacroAssembler _masm(&cbuf);
 3011     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3012                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3013   %}
 3014 
 3015   // This encoding class is generated automatically from ad_encode.m4.
 3016   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3017   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3018     Register src_reg = as_Register($src$$reg);
 3019     // we sometimes get asked to store the stack pointer into the
 3020     // current thread -- we cannot do that directly on AArch64
 3021     if (src_reg == r31_sp) {
 3022       C2_MacroAssembler _masm(&cbuf);
 3023       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3024       __ mov(rscratch2, sp);
 3025       src_reg = rscratch2;
 3026     }
 3027     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3028                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3029   %}
 3030 
 3031   // This encoding class is generated automatically from ad_encode.m4.
 3032   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3033   enc_class aarch64_enc_str0(memory8 mem) %{
 3034     C2_MacroAssembler _masm(&cbuf);
 3035     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3036                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3037   %}
 3038 
 3039   // This encoding class is generated automatically from ad_encode.m4.
 3040   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3041   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3042     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3043     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3044                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3045   %}
 3046 
 3047   // This encoding class is generated automatically from ad_encode.m4.
 3048   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3049   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3050     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3051     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3052                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3053   %}
 3054 
 3055   // This encoding class is generated automatically from ad_encode.m4.
 3056   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3057   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3058       C2_MacroAssembler _masm(&cbuf);
 3059       __ membar(Assembler::StoreStore);
 3060       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3061                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3062   %}
 3063 
 3064   // END Non-volatile memory access
 3065 
 3066   // Vector loads and stores
 3067   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3068     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3069     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3070        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3071   %}
 3072 
 3073   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3074     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3075     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3076        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3077   %}
 3078 
 3079   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3080     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3081     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3082        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3086     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3088        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3092     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3094        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3098     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3100        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3101   %}
 3102 
 3103   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3104     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3105     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3106        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3110     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3111     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3112        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3113   %}
 3114 
 3115   // volatile loads and stores
 3116 
 3117   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3118     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3119                  rscratch1, stlrb);
 3120   %}
 3121 
 3122   enc_class aarch64_enc_stlrb0(memory mem) %{
 3123     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3124                  rscratch1, stlrb);
 3125   %}
 3126 
 3127   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3128     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3129                  rscratch1, stlrh);
 3130   %}
 3131 
 3132   enc_class aarch64_enc_stlrh0(memory mem) %{
 3133     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3134                  rscratch1, stlrh);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3138     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3139                  rscratch1, stlrw);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_stlrw0(memory mem) %{
 3143     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3144                  rscratch1, stlrw);
 3145   %}
 3146 
 3147   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3148     Register dst_reg = as_Register($dst$$reg);
 3149     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3150              rscratch1, ldarb);
 3151     __ sxtbw(dst_reg, dst_reg);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3155     Register dst_reg = as_Register($dst$$reg);
 3156     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3157              rscratch1, ldarb);
 3158     __ sxtb(dst_reg, dst_reg);
 3159   %}
 3160 
 3161   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3162     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3163              rscratch1, ldarb);
 3164   %}
 3165 
 3166   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3167     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3168              rscratch1, ldarb);
 3169   %}
 3170 
 3171   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3172     Register dst_reg = as_Register($dst$$reg);
 3173     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3174              rscratch1, ldarh);
 3175     __ sxthw(dst_reg, dst_reg);
 3176   %}
 3177 
 3178   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3179     Register dst_reg = as_Register($dst$$reg);
 3180     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3181              rscratch1, ldarh);
 3182     __ sxth(dst_reg, dst_reg);
 3183   %}
 3184 
 3185   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3186     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3187              rscratch1, ldarh);
 3188   %}
 3189 
 3190   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3191     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3192              rscratch1, ldarh);
 3193   %}
 3194 
 3195   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3196     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3197              rscratch1, ldarw);
 3198   %}
 3199 
 3200   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3201     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3202              rscratch1, ldarw);
 3203   %}
 3204 
 3205   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3206     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3207              rscratch1, ldar);
 3208   %}
 3209 
 3210   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3211     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212              rscratch1, ldarw);
 3213     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3214   %}
 3215 
 3216   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3217     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3218              rscratch1, ldar);
 3219     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3220   %}
 3221 
 3222   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3223     Register src_reg = as_Register($src$$reg);
 3224     // we sometimes get asked to store the stack pointer into the
 3225     // current thread -- we cannot do that directly on AArch64
 3226     if (src_reg == r31_sp) {
 3227       C2_MacroAssembler _masm(&cbuf);
 3228       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3229       __ mov(rscratch2, sp);
 3230       src_reg = rscratch2;
 3231     }
 3232     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3233                  rscratch1, stlr);
 3234   %}
 3235 
 3236   enc_class aarch64_enc_stlr0(memory mem) %{
 3237     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3238                  rscratch1, stlr);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3242     {
 3243       C2_MacroAssembler _masm(&cbuf);
 3244       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3245       __ fmovs(rscratch2, src_reg);
 3246     }
 3247     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248                  rscratch1, stlrw);
 3249   %}
 3250 
 3251   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3252     {
 3253       C2_MacroAssembler _masm(&cbuf);
 3254       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3255       __ fmovd(rscratch2, src_reg);
 3256     }
 3257     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3258                  rscratch1, stlr);
 3259   %}
 3260 
 3261   // synchronized read/update encodings
 3262 
 3263   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3264     C2_MacroAssembler _masm(&cbuf);
 3265     Register dst_reg = as_Register($dst$$reg);
 3266     Register base = as_Register($mem$$base);
 3267     int index = $mem$$index;
 3268     int scale = $mem$$scale;
 3269     int disp = $mem$$disp;
 3270     if (index == -1) {
 3271        if (disp != 0) {
 3272         __ lea(rscratch1, Address(base, disp));
 3273         __ ldaxr(dst_reg, rscratch1);
 3274       } else {
 3275         // TODO
 3276         // should we ever get anything other than this case?
 3277         __ ldaxr(dst_reg, base);
 3278       }
 3279     } else {
 3280       Register index_reg = as_Register(index);
 3281       if (disp == 0) {
 3282         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3283         __ ldaxr(dst_reg, rscratch1);
 3284       } else {
 3285         __ lea(rscratch1, Address(base, disp));
 3286         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3287         __ ldaxr(dst_reg, rscratch1);
 3288       }
 3289     }
 3290   %}
 3291 
 3292   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3293     C2_MacroAssembler _masm(&cbuf);
 3294     Register src_reg = as_Register($src$$reg);
 3295     Register base = as_Register($mem$$base);
 3296     int index = $mem$$index;
 3297     int scale = $mem$$scale;
 3298     int disp = $mem$$disp;
 3299     if (index == -1) {
 3300        if (disp != 0) {
 3301         __ lea(rscratch2, Address(base, disp));
 3302         __ stlxr(rscratch1, src_reg, rscratch2);
 3303       } else {
 3304         // TODO
 3305         // should we ever get anything other than this case?
 3306         __ stlxr(rscratch1, src_reg, base);
 3307       }
 3308     } else {
 3309       Register index_reg = as_Register(index);
 3310       if (disp == 0) {
 3311         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3312         __ stlxr(rscratch1, src_reg, rscratch2);
 3313       } else {
 3314         __ lea(rscratch2, Address(base, disp));
 3315         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3316         __ stlxr(rscratch1, src_reg, rscratch2);
 3317       }
 3318     }
 3319     __ cmpw(rscratch1, zr);
 3320   %}
 3321 
 3322   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3323     C2_MacroAssembler _masm(&cbuf);
 3324     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3325     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3326                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3327                /*weak*/ false, noreg);
 3328   %}
 3329 
 3330   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3331     C2_MacroAssembler _masm(&cbuf);
 3332     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3333     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3334                Assembler::word, /*acquire*/ false, /*release*/ true,
 3335                /*weak*/ false, noreg);
 3336   %}
 3337 
 3338   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3339     C2_MacroAssembler _masm(&cbuf);
 3340     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3341     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3342                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3343                /*weak*/ false, noreg);
 3344   %}
 3345 
 3346   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3347     C2_MacroAssembler _masm(&cbuf);
 3348     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3349     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3350                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3351                /*weak*/ false, noreg);
 3352   %}
 3353 
 3354 
 3355   // The only difference between aarch64_enc_cmpxchg and
 3356   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3357   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3358   // lock.
 3359   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3360     C2_MacroAssembler _masm(&cbuf);
 3361     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3362     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3363                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3364                /*weak*/ false, noreg);
 3365   %}
 3366 
 3367   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3368     C2_MacroAssembler _masm(&cbuf);
 3369     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3370     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3371                Assembler::word, /*acquire*/ true, /*release*/ true,
 3372                /*weak*/ false, noreg);
 3373   %}
 3374 
 3375   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3376     C2_MacroAssembler _masm(&cbuf);
 3377     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3378     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3379                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3380                /*weak*/ false, noreg);
 3381   %}
 3382 
 3383   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3384     C2_MacroAssembler _masm(&cbuf);
 3385     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3386     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3387                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3388                /*weak*/ false, noreg);
 3389   %}
 3390 
 3391   // auxiliary used for CompareAndSwapX to set result register
 3392   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3393     C2_MacroAssembler _masm(&cbuf);
 3394     Register res_reg = as_Register($res$$reg);
 3395     __ cset(res_reg, Assembler::EQ);
 3396   %}
 3397 
 3398   // prefetch encodings
 3399 
 3400   enc_class aarch64_enc_prefetchw(memory mem) %{
 3401     C2_MacroAssembler _masm(&cbuf);
 3402     Register base = as_Register($mem$$base);
 3403     int index = $mem$$index;
 3404     int scale = $mem$$scale;
 3405     int disp = $mem$$disp;
 3406     if (index == -1) {
 3407       __ prfm(Address(base, disp), PSTL1KEEP);
 3408     } else {
 3409       Register index_reg = as_Register(index);
 3410       if (disp == 0) {
 3411         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3412       } else {
 3413         __ lea(rscratch1, Address(base, disp));
 3414 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3415       }
 3416     }
 3417   %}
 3418 
 3419   /// mov envcodings
 3420 
 3421   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3422     C2_MacroAssembler _masm(&cbuf);
 3423     uint32_t con = (uint32_t)$src$$constant;
 3424     Register dst_reg = as_Register($dst$$reg);
 3425     if (con == 0) {
 3426       __ movw(dst_reg, zr);
 3427     } else {
 3428       __ movw(dst_reg, con);
 3429     }
 3430   %}
 3431 
 3432   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3433     C2_MacroAssembler _masm(&cbuf);
 3434     Register dst_reg = as_Register($dst$$reg);
 3435     uint64_t con = (uint64_t)$src$$constant;
 3436     if (con == 0) {
 3437       __ mov(dst_reg, zr);
 3438     } else {
 3439       __ mov(dst_reg, con);
 3440     }
 3441   %}
 3442 
 3443   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3444     C2_MacroAssembler _masm(&cbuf);
 3445     Register dst_reg = as_Register($dst$$reg);
 3446     address con = (address)$src$$constant;
 3447     if (con == NULL || con == (address)1) {
 3448       ShouldNotReachHere();
 3449     } else {
 3450       relocInfo::relocType rtype = $src->constant_reloc();
 3451       if (rtype == relocInfo::oop_type) {
 3452         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3453       } else if (rtype == relocInfo::metadata_type) {
 3454         __ mov_metadata(dst_reg, (Metadata*)con);
 3455       } else {
 3456         assert(rtype == relocInfo::none, "unexpected reloc type");
 3457         if (! __ is_valid_AArch64_address(con) ||
 3458             con < (address)(uintptr_t)os::vm_page_size()) {
 3459           __ mov(dst_reg, con);
 3460         } else {
 3461           uint64_t offset;
 3462           __ adrp(dst_reg, con, offset);
 3463           __ add(dst_reg, dst_reg, offset);
 3464         }
 3465       }
 3466     }
 3467   %}
 3468 
 3469   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3470     C2_MacroAssembler _masm(&cbuf);
 3471     Register dst_reg = as_Register($dst$$reg);
 3472     __ mov(dst_reg, zr);
 3473   %}
 3474 
 3475   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3476     C2_MacroAssembler _masm(&cbuf);
 3477     Register dst_reg = as_Register($dst$$reg);
 3478     __ mov(dst_reg, (uint64_t)1);
 3479   %}
 3480 
 3481   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3482     C2_MacroAssembler _masm(&cbuf);
 3483     __ load_byte_map_base($dst$$Register);
 3484   %}
 3485 
 3486   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3487     C2_MacroAssembler _masm(&cbuf);
 3488     Register dst_reg = as_Register($dst$$reg);
 3489     address con = (address)$src$$constant;
 3490     if (con == NULL) {
 3491       ShouldNotReachHere();
 3492     } else {
 3493       relocInfo::relocType rtype = $src->constant_reloc();
 3494       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3495       __ set_narrow_oop(dst_reg, (jobject)con);
 3496     }
 3497   %}
 3498 
 3499   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3500     C2_MacroAssembler _masm(&cbuf);
 3501     Register dst_reg = as_Register($dst$$reg);
 3502     __ mov(dst_reg, zr);
 3503   %}
 3504 
 3505   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3506     C2_MacroAssembler _masm(&cbuf);
 3507     Register dst_reg = as_Register($dst$$reg);
 3508     address con = (address)$src$$constant;
 3509     if (con == NULL) {
 3510       ShouldNotReachHere();
 3511     } else {
 3512       relocInfo::relocType rtype = $src->constant_reloc();
 3513       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3514       __ set_narrow_klass(dst_reg, (Klass *)con);
 3515     }
 3516   %}
 3517 
 3518   // arithmetic encodings
 3519 
 3520   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3521     C2_MacroAssembler _masm(&cbuf);
 3522     Register dst_reg = as_Register($dst$$reg);
 3523     Register src_reg = as_Register($src1$$reg);
 3524     int32_t con = (int32_t)$src2$$constant;
 3525     // add has primary == 0, subtract has primary == 1
 3526     if ($primary) { con = -con; }
 3527     if (con < 0) {
 3528       __ subw(dst_reg, src_reg, -con);
 3529     } else {
 3530       __ addw(dst_reg, src_reg, con);
 3531     }
 3532   %}
 3533 
 3534   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3535     C2_MacroAssembler _masm(&cbuf);
 3536     Register dst_reg = as_Register($dst$$reg);
 3537     Register src_reg = as_Register($src1$$reg);
 3538     int32_t con = (int32_t)$src2$$constant;
 3539     // add has primary == 0, subtract has primary == 1
 3540     if ($primary) { con = -con; }
 3541     if (con < 0) {
 3542       __ sub(dst_reg, src_reg, -con);
 3543     } else {
 3544       __ add(dst_reg, src_reg, con);
 3545     }
 3546   %}
 3547 
 3548   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3549     C2_MacroAssembler _masm(&cbuf);
 3550    Register dst_reg = as_Register($dst$$reg);
 3551    Register src1_reg = as_Register($src1$$reg);
 3552    Register src2_reg = as_Register($src2$$reg);
 3553     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3554   %}
 3555 
 3556   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558    Register dst_reg = as_Register($dst$$reg);
 3559    Register src1_reg = as_Register($src1$$reg);
 3560    Register src2_reg = as_Register($src2$$reg);
 3561     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3565     C2_MacroAssembler _masm(&cbuf);
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3573     C2_MacroAssembler _masm(&cbuf);
 3574    Register dst_reg = as_Register($dst$$reg);
 3575    Register src1_reg = as_Register($src1$$reg);
 3576    Register src2_reg = as_Register($src2$$reg);
 3577     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3578   %}
 3579 
 3580   // compare instruction encodings
 3581 
 3582   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3583     C2_MacroAssembler _masm(&cbuf);
 3584     Register reg1 = as_Register($src1$$reg);
 3585     Register reg2 = as_Register($src2$$reg);
 3586     __ cmpw(reg1, reg2);
 3587   %}
 3588 
 3589   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3590     C2_MacroAssembler _masm(&cbuf);
 3591     Register reg = as_Register($src1$$reg);
 3592     int32_t val = $src2$$constant;
 3593     if (val >= 0) {
 3594       __ subsw(zr, reg, val);
 3595     } else {
 3596       __ addsw(zr, reg, -val);
 3597     }
 3598   %}
 3599 
 3600   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3601     C2_MacroAssembler _masm(&cbuf);
 3602     Register reg1 = as_Register($src1$$reg);
 3603     uint32_t val = (uint32_t)$src2$$constant;
 3604     __ movw(rscratch1, val);
 3605     __ cmpw(reg1, rscratch1);
 3606   %}
 3607 
 3608   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3609     C2_MacroAssembler _masm(&cbuf);
 3610     Register reg1 = as_Register($src1$$reg);
 3611     Register reg2 = as_Register($src2$$reg);
 3612     __ cmp(reg1, reg2);
 3613   %}
 3614 
 3615   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3616     C2_MacroAssembler _masm(&cbuf);
 3617     Register reg = as_Register($src1$$reg);
 3618     int64_t val = $src2$$constant;
 3619     if (val >= 0) {
 3620       __ subs(zr, reg, val);
 3621     } else if (val != -val) {
 3622       __ adds(zr, reg, -val);
 3623     } else {
 3624     // aargh, Long.MIN_VALUE is a special case
 3625       __ orr(rscratch1, zr, (uint64_t)val);
 3626       __ subs(zr, reg, rscratch1);
 3627     }
 3628   %}
 3629 
 3630   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3631     C2_MacroAssembler _masm(&cbuf);
 3632     Register reg1 = as_Register($src1$$reg);
 3633     uint64_t val = (uint64_t)$src2$$constant;
 3634     __ mov(rscratch1, val);
 3635     __ cmp(reg1, rscratch1);
 3636   %}
 3637 
 3638   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3639     C2_MacroAssembler _masm(&cbuf);
 3640     Register reg1 = as_Register($src1$$reg);
 3641     Register reg2 = as_Register($src2$$reg);
 3642     __ cmp(reg1, reg2);
 3643   %}
 3644 
 3645   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3646     C2_MacroAssembler _masm(&cbuf);
 3647     Register reg1 = as_Register($src1$$reg);
 3648     Register reg2 = as_Register($src2$$reg);
 3649     __ cmpw(reg1, reg2);
 3650   %}
 3651 
 3652   enc_class aarch64_enc_testp(iRegP src) %{
 3653     C2_MacroAssembler _masm(&cbuf);
 3654     Register reg = as_Register($src$$reg);
 3655     __ cmp(reg, zr);
 3656   %}
 3657 
 3658   enc_class aarch64_enc_testn(iRegN src) %{
 3659     C2_MacroAssembler _masm(&cbuf);
 3660     Register reg = as_Register($src$$reg);
 3661     __ cmpw(reg, zr);
 3662   %}
 3663 
 3664   enc_class aarch64_enc_b(label lbl) %{
 3665     C2_MacroAssembler _masm(&cbuf);
 3666     Label *L = $lbl$$label;
 3667     __ b(*L);
 3668   %}
 3669 
 3670   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3671     C2_MacroAssembler _masm(&cbuf);
 3672     Label *L = $lbl$$label;
 3673     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3674   %}
 3675 
 3676   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3677     C2_MacroAssembler _masm(&cbuf);
 3678     Label *L = $lbl$$label;
 3679     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3680   %}
 3681 
 3682   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3683   %{
 3684      Register sub_reg = as_Register($sub$$reg);
 3685      Register super_reg = as_Register($super$$reg);
 3686      Register temp_reg = as_Register($temp$$reg);
 3687      Register result_reg = as_Register($result$$reg);
 3688 
 3689      Label miss;
 3690      C2_MacroAssembler _masm(&cbuf);
 3691      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3692                                      NULL, &miss,
 3693                                      /*set_cond_codes:*/ true);
 3694      if ($primary) {
 3695        __ mov(result_reg, zr);
 3696      }
 3697      __ bind(miss);
 3698   %}
 3699 
 3700   enc_class aarch64_enc_java_static_call(method meth) %{
 3701     C2_MacroAssembler _masm(&cbuf);
 3702 
 3703     address addr = (address)$meth$$method;
 3704     address call;
 3705     if (!_method) {
 3706       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3707       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3708       if (call == NULL) {
 3709         ciEnv::current()->record_failure("CodeCache is full");
 3710         return;
 3711       }
 3712     } else {
 3713       int method_index = resolved_method_index(cbuf);
 3714       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3715                                                   : static_call_Relocation::spec(method_index);
 3716       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3717       if (call == NULL) {
 3718         ciEnv::current()->record_failure("CodeCache is full");
 3719         return;
 3720       }
 3721       // Emit stub for static call
 3722       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3723       if (stub == NULL) {
 3724         ciEnv::current()->record_failure("CodeCache is full");
 3725         return;
 3726       }
 3727     }
 3728 
 3729     // Only non uncommon_trap calls need to reinitialize ptrue.
 3730     if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
 3731       __ reinitialize_ptrue();
 3732     }
 3733   %}
 3734 
 3735   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3736     C2_MacroAssembler _masm(&cbuf);
 3737     int method_index = resolved_method_index(cbuf);
 3738     address call = __ ic_call((address)$meth$$method, method_index);
 3739     if (call == NULL) {
 3740       ciEnv::current()->record_failure("CodeCache is full");
 3741       return;
 3742     } else if (Compile::current()->max_vector_size() >= 16) {
 3743       __ reinitialize_ptrue();
 3744     }
 3745   %}
 3746 
 3747   enc_class aarch64_enc_call_epilog() %{
 3748     C2_MacroAssembler _masm(&cbuf);
 3749     if (VerifyStackAtCalls) {
 3750       // Check that stack depth is unchanged: find majik cookie on stack
 3751       __ call_Unimplemented();
 3752     }
 3753   %}
 3754 
 3755   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3756     C2_MacroAssembler _masm(&cbuf);
 3757 
 3758     // some calls to generated routines (arraycopy code) are scheduled
 3759     // by C2 as runtime calls. if so we can call them using a br (they
 3760     // will be in a reachable segment) otherwise we have to use a blr
 3761     // which loads the absolute address into a register.
 3762     address entry = (address)$meth$$method;
 3763     CodeBlob *cb = CodeCache::find_blob(entry);
 3764     if (cb) {
 3765       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3766       if (call == NULL) {
 3767         ciEnv::current()->record_failure("CodeCache is full");
 3768         return;
 3769       }
 3770     } else {
 3771       Label retaddr;
 3772       __ adr(rscratch2, retaddr);
 3773       __ lea(rscratch1, RuntimeAddress(entry));
 3774       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3775       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3776       __ blr(rscratch1);
 3777       __ bind(retaddr);
 3778       __ add(sp, sp, 2 * wordSize);
 3779     }
 3780     if (Compile::current()->max_vector_size() >= 16) {
 3781       __ reinitialize_ptrue();
 3782     }
 3783   %}
 3784 
 3785   enc_class aarch64_enc_rethrow() %{
 3786     C2_MacroAssembler _masm(&cbuf);
 3787     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3788   %}
 3789 
 3790   enc_class aarch64_enc_ret() %{
 3791     C2_MacroAssembler _masm(&cbuf);
 3792 #ifdef ASSERT
 3793     if (Compile::current()->max_vector_size() >= 16) {
 3794       __ verify_ptrue();
 3795     }
 3796 #endif
 3797     __ ret(lr);
 3798   %}
 3799 
 3800   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3801     C2_MacroAssembler _masm(&cbuf);
 3802     Register target_reg = as_Register($jump_target$$reg);
 3803     __ br(target_reg);
 3804   %}
 3805 
 3806   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3807     C2_MacroAssembler _masm(&cbuf);
 3808     Register target_reg = as_Register($jump_target$$reg);
 3809     // exception oop should be in r0
 3810     // ret addr has been popped into lr
 3811     // callee expects it in r3
 3812     __ mov(r3, lr);
 3813     __ br(target_reg);
 3814   %}
 3815 
 3816   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3817     C2_MacroAssembler _masm(&cbuf);
 3818     Register oop = as_Register($object$$reg);
 3819     Register box = as_Register($box$$reg);
 3820     Register disp_hdr = as_Register($tmp$$reg);
 3821     Register tmp = as_Register($tmp2$$reg);
 3822     Label cont;
 3823     Label object_has_monitor;
 3824     Label cas_failed;
 3825 
 3826     assert_different_registers(oop, box, tmp, disp_hdr);
 3827 
 3828     // Load markWord from object into displaced_header.
 3829     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3830 
 3831     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3832       __ load_klass(tmp, oop);
 3833       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3834       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3835       __ br(Assembler::NE, cont);
 3836     }
 3837 
 3838     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3839       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
 3840     }
 3841 
 3842     // Check for existing monitor
 3843     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3844 
 3845     if (UseFastLocking) {
 3846       __ fast_lock(oop, disp_hdr, tmp, rscratch1, cont, false);
 3847       // Indicate success at cont.
 3848       __ cmp(oop, oop);
 3849     } else {
 3850       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3851       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3852 
 3853       // Initialize the box. (Must happen before we update the object mark!)
 3854       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3855 
 3856       // Compare object markWord with an unlocked value (tmp) and if
 3857       // equal exchange the stack address of our box with object markWord.
 3858       // On failure disp_hdr contains the possibly locked markWord.
 3859       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3860                  /*release*/ true, /*weak*/ false, disp_hdr);
 3861       __ br(Assembler::EQ, cont);
 3862 
 3863       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3864 
 3865       // If the compare-and-exchange succeeded, then we found an unlocked
 3866       // object, will have now locked it will continue at label cont
 3867 
 3868       __ bind(cas_failed);
 3869       // We did not see an unlocked object so try the fast recursive case.
 3870 
 3871       // Check if the owner is self by comparing the value in the
 3872       // markWord of object (disp_hdr) with the stack pointer.
 3873       __ mov(rscratch1, sp);
 3874       __ sub(disp_hdr, disp_hdr, rscratch1);
 3875       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3876       // If condition is true we are cont and hence we can store 0 as the
 3877       // displaced header in the box, which indicates that it is a recursive lock.
 3878       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3879       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3880     }
 3881     __ b(cont);
 3882 
 3883     // Handle existing monitor.
 3884     __ bind(object_has_monitor);
 3885 
 3886     // The object's monitor m is unlocked iff m->owner == NULL,
 3887     // otherwise m->owner may contain a thread or a stack address.
 3888     //
 3889     // Try to CAS m->owner from NULL to current thread.
 3890     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3891     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3892                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3893 
 3894     if (!UseFastLocking) {
 3895       // Store a non-null value into the box to avoid looking like a re-entrant
 3896       // lock. The fast-path monitor unlock code checks for
 3897       // markWord::monitor_value so use markWord::unused_mark which has the
 3898       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3899       __ mov(tmp, (address)markWord::unused_mark().value());
 3900       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3901     }
 3902     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3903 
 3904     __ cmp(rscratch1, rthread);
 3905     __ br(Assembler::NE, cont); // Check for recursive locking
 3906 
 3907     // Recursive lock case
 3908     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3909     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3910 
 3911     __ bind(cont);
 3912     // flag == EQ indicates success
 3913     // flag == NE indicates failure
 3914   %}
 3915 
 3916   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3917     C2_MacroAssembler _masm(&cbuf);
 3918     Register oop = as_Register($object$$reg);
 3919     Register box = as_Register($box$$reg);
 3920     Register disp_hdr = as_Register($tmp$$reg);
 3921     Register tmp = as_Register($tmp2$$reg);
 3922     Label cont;
 3923     Label object_has_monitor;
 3924 
 3925     assert_different_registers(oop, box, tmp, disp_hdr);
 3926 
 3927     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3928       __ biased_locking_exit(oop, tmp, cont);
 3929     }
 3930 
 3931     if (!UseFastLocking) {
 3932       // Find the lock address and load the displaced header from the stack.
 3933       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3934 
 3935       // If the displaced header is 0, we have a recursive unlock.
 3936       __ cmp(disp_hdr, zr);
 3937       __ br(Assembler::EQ, cont);
 3938     }
 3939 
 3940     // Handle existing monitor.
 3941     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3942     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3943 
 3944 
 3945     if (UseFastLocking) {
 3946       __ fast_unlock(oop, tmp, box, disp_hdr, cont);
 3947       // Indicate success at cont.
 3948       __ cmp(oop, oop);
 3949     } else {
 3950       // Check if it is still a light weight lock, this is is true if we
 3951       // see the stack address of the basicLock in the markWord of the
 3952       // object.
 3953 
 3954       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3955                  /*release*/ true, /*weak*/ false, tmp);
 3956     }
 3957     __ b(cont);
 3958     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3959 
 3960     // Handle existing monitor.
 3961     __ bind(object_has_monitor);
 3962     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3963     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3964 
 3965     if (UseFastLocking) {
 3966       // If the owner is anonymous, we need to fix it -- in an outline stub.
 3967       Register tmp2 = disp_hdr;
 3968       __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3969       // We cannot use tbnz here, the target might be too far away and cannot
 3970       // be encoded.
 3971       __ tst(tmp2, (uint64_t)(intptr_t) ANONYMOUS_OWNER);
 3972       C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 3973       Compile::current()->output()->add_stub(stub);
 3974       __ br(Assembler::NE, stub->entry());
 3975       __ bind(stub->continuation());
 3976     }
 3977 
 3978     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3979 
 3980     Label notRecursive;
 3981     __ cbz(disp_hdr, notRecursive);
 3982 
 3983     // Recursive lock
 3984     __ sub(disp_hdr, disp_hdr, 1u);
 3985     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3986     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3987     __ b(cont);
 3988 
 3989     __ bind(notRecursive);
 3990     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3991     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3992     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3993     __ cmp(rscratch1, zr); // Sets flags for result
 3994     __ cbnz(rscratch1, cont);
 3995     // need a release store here
 3996     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3997     __ stlr(zr, tmp); // set unowned
 3998 
 3999     __ bind(cont);
 4000     // flag == EQ indicates success
 4001     // flag == NE indicates failure
 4002   %}
 4003 
 4004 %}
 4005 
 4006 //----------FRAME--------------------------------------------------------------
 4007 // Definition of frame structure and management information.
 4008 //
 4009 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4010 //                             |   (to get allocators register number
 4011 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4012 //  r   CALLER     |        |
 4013 //  o     |        +--------+      pad to even-align allocators stack-slot
 4014 //  w     V        |  pad0  |        numbers; owned by CALLER
 4015 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4016 //  h     ^        |   in   |  5
 4017 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4018 //  |     |        |        |  3
 4019 //  |     |        +--------+
 4020 //  V     |        | old out|      Empty on Intel, window on Sparc
 4021 //        |    old |preserve|      Must be even aligned.
 4022 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4023 //        |        |   in   |  3   area for Intel ret address
 4024 //     Owned by    |preserve|      Empty on Sparc.
 4025 //       SELF      +--------+
 4026 //        |        |  pad2  |  2   pad to align old SP
 4027 //        |        +--------+  1
 4028 //        |        | locks  |  0
 4029 //        |        +--------+----> OptoReg::stack0(), even aligned
 4030 //        |        |  pad1  | 11   pad to align new SP
 4031 //        |        +--------+
 4032 //        |        |        | 10
 4033 //        |        | spills |  9   spills
 4034 //        V        |        |  8   (pad0 slot for callee)
 4035 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4036 //        ^        |  out   |  7
 4037 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4038 //     Owned by    +--------+
 4039 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4040 //        |    new |preserve|      Must be even-aligned.
 4041 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4042 //        |        |        |
 4043 //
 4044 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4045 //         known from SELF's arguments and the Java calling convention.
 4046 //         Region 6-7 is determined per call site.
 4047 // Note 2: If the calling convention leaves holes in the incoming argument
 4048 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4049 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4050 //         incoming area, as the Java calling convention is completely under
 4051 //         the control of the AD file.  Doubles can be sorted and packed to
 4052 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4053 //         varargs C calling conventions.
 4054 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4055 //         even aligned with pad0 as needed.
 4056 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4057 //           (the latter is true on Intel but is it false on AArch64?)
 4058 //         region 6-11 is even aligned; it may be padded out more so that
 4059 //         the region from SP to FP meets the minimum stack alignment.
 4060 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4061 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4062 //         SP meets the minimum alignment.
 4063 
 4064 frame %{
 4065   // These three registers define part of the calling convention
 4066   // between compiled code and the interpreter.
 4067 
 4068   // Inline Cache Register or Method for I2C.
 4069   inline_cache_reg(R12);
 4070 
 4071   // Number of stack slots consumed by locking an object
 4072   sync_stack_slots(2);
 4073 
 4074   // Compiled code's Frame Pointer
 4075   frame_pointer(R31);
 4076 
 4077   // Interpreter stores its frame pointer in a register which is
 4078   // stored to the stack by I2CAdaptors.
 4079   // I2CAdaptors convert from interpreted java to compiled java.
 4080   interpreter_frame_pointer(R29);
 4081 
 4082   // Stack alignment requirement
 4083   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4084 
 4085   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4086   // for calls to C.  Supports the var-args backing area for register parms.
 4087   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4088 
 4089   // The after-PROLOG location of the return address.  Location of
 4090   // return address specifies a type (REG or STACK) and a number
 4091   // representing the register number (i.e. - use a register name) or
 4092   // stack slot.
 4093   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4094   // Otherwise, it is above the locks and verification slot and alignment word
 4095   // TODO this may well be correct but need to check why that - 2 is there
 4096   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4097   // which folds in the space used for monitors
 4098   return_addr(STACK - 2 +
 4099               align_up((Compile::current()->in_preserve_stack_slots() +
 4100                         Compile::current()->fixed_slots()),
 4101                        stack_alignment_in_slots()));
 4102 
 4103   // Location of compiled Java return values.  Same as C for now.
 4104   return_value
 4105   %{
 4106     // TODO do we allow ideal_reg == Op_RegN???
 4107     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4108            "only return normal values");
 4109 
 4110     static const int lo[Op_RegL + 1] = { // enum name
 4111       0,                                 // Op_Node
 4112       0,                                 // Op_Set
 4113       R0_num,                            // Op_RegN
 4114       R0_num,                            // Op_RegI
 4115       R0_num,                            // Op_RegP
 4116       V0_num,                            // Op_RegF
 4117       V0_num,                            // Op_RegD
 4118       R0_num                             // Op_RegL
 4119     };
 4120 
 4121     static const int hi[Op_RegL + 1] = { // enum name
 4122       0,                                 // Op_Node
 4123       0,                                 // Op_Set
 4124       OptoReg::Bad,                      // Op_RegN
 4125       OptoReg::Bad,                      // Op_RegI
 4126       R0_H_num,                          // Op_RegP
 4127       OptoReg::Bad,                      // Op_RegF
 4128       V0_H_num,                          // Op_RegD
 4129       R0_H_num                           // Op_RegL
 4130     };
 4131 
 4132     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4133   %}
 4134 %}
 4135 
 4136 //----------ATTRIBUTES---------------------------------------------------------
 4137 //----------Operand Attributes-------------------------------------------------
 4138 op_attrib op_cost(1);        // Required cost attribute
 4139 
 4140 //----------Instruction Attributes---------------------------------------------
 4141 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4142 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4143 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4144                                 // a non-matching short branch variant
 4145                                 // of some long branch?
 4146 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4147                                 // be a power of 2) specifies the
 4148                                 // alignment that some part of the
 4149                                 // instruction (not necessarily the
 4150                                 // start) requires.  If > 1, a
 4151                                 // compute_padding() function must be
 4152                                 // provided for the instruction
 4153 
 4154 //----------OPERANDS-----------------------------------------------------------
 4155 // Operand definitions must precede instruction definitions for correct parsing
 4156 // in the ADLC because operands constitute user defined types which are used in
 4157 // instruction definitions.
 4158 
 4159 //----------Simple Operands----------------------------------------------------
 4160 
 4161 // Integer operands 32 bit
 4162 // 32 bit immediate
 4163 operand immI()
 4164 %{
 4165   match(ConI);
 4166 
 4167   op_cost(0);
 4168   format %{ %}
 4169   interface(CONST_INTER);
 4170 %}
 4171 
 4172 // 32 bit zero
 4173 operand immI0()
 4174 %{
 4175   predicate(n->get_int() == 0);
 4176   match(ConI);
 4177 
 4178   op_cost(0);
 4179   format %{ %}
 4180   interface(CONST_INTER);
 4181 %}
 4182 
 4183 // 32 bit unit increment
 4184 operand immI_1()
 4185 %{
 4186   predicate(n->get_int() == 1);
 4187   match(ConI);
 4188 
 4189   op_cost(0);
 4190   format %{ %}
 4191   interface(CONST_INTER);
 4192 %}
 4193 
 4194 // 32 bit unit decrement
 4195 operand immI_M1()
 4196 %{
 4197   predicate(n->get_int() == -1);
 4198   match(ConI);
 4199 
 4200   op_cost(0);
 4201   format %{ %}
 4202   interface(CONST_INTER);
 4203 %}
 4204 
 4205 // Shift values for add/sub extension shift
 4206 operand immIExt()
 4207 %{
 4208   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4209   match(ConI);
 4210 
 4211   op_cost(0);
 4212   format %{ %}
 4213   interface(CONST_INTER);
 4214 %}
 4215 
 4216 operand immI_le_4()
 4217 %{
 4218   predicate(n->get_int() <= 4);
 4219   match(ConI);
 4220 
 4221   op_cost(0);
 4222   format %{ %}
 4223   interface(CONST_INTER);
 4224 %}
 4225 
 4226 operand immI_31()
 4227 %{
 4228   predicate(n->get_int() == 31);
 4229   match(ConI);
 4230 
 4231   op_cost(0);
 4232   format %{ %}
 4233   interface(CONST_INTER);
 4234 %}
 4235 
 4236 operand immI_2()
 4237 %{
 4238   predicate(n->get_int() == 2);
 4239   match(ConI);
 4240 
 4241   op_cost(0);
 4242   format %{ %}
 4243   interface(CONST_INTER);
 4244 %}
 4245 
 4246 operand immI_4()
 4247 %{
 4248   predicate(n->get_int() == 4);
 4249   match(ConI);
 4250 
 4251   op_cost(0);
 4252   format %{ %}
 4253   interface(CONST_INTER);
 4254 %}
 4255 
 4256 operand immI_8()
 4257 %{
 4258   predicate(n->get_int() == 8);
 4259   match(ConI);
 4260 
 4261   op_cost(0);
 4262   format %{ %}
 4263   interface(CONST_INTER);
 4264 %}
 4265 
 4266 operand immI_16()
 4267 %{
 4268   predicate(n->get_int() == 16);
 4269   match(ConI);
 4270 
 4271   op_cost(0);
 4272   format %{ %}
 4273   interface(CONST_INTER);
 4274 %}
 4275 
 4276 operand immI_24()
 4277 %{
 4278   predicate(n->get_int() == 24);
 4279   match(ConI);
 4280 
 4281   op_cost(0);
 4282   format %{ %}
 4283   interface(CONST_INTER);
 4284 %}
 4285 
 4286 operand immI_32()
 4287 %{
 4288   predicate(n->get_int() == 32);
 4289   match(ConI);
 4290 
 4291   op_cost(0);
 4292   format %{ %}
 4293   interface(CONST_INTER);
 4294 %}
 4295 
 4296 operand immI_48()
 4297 %{
 4298   predicate(n->get_int() == 48);
 4299   match(ConI);
 4300 
 4301   op_cost(0);
 4302   format %{ %}
 4303   interface(CONST_INTER);
 4304 %}
 4305 
 4306 operand immI_56()
 4307 %{
 4308   predicate(n->get_int() == 56);
 4309   match(ConI);
 4310 
 4311   op_cost(0);
 4312   format %{ %}
 4313   interface(CONST_INTER);
 4314 %}
 4315 
 4316 operand immI_63()
 4317 %{
 4318   predicate(n->get_int() == 63);
 4319   match(ConI);
 4320 
 4321   op_cost(0);
 4322   format %{ %}
 4323   interface(CONST_INTER);
 4324 %}
 4325 
 4326 operand immI_64()
 4327 %{
 4328   predicate(n->get_int() == 64);
 4329   match(ConI);
 4330 
 4331   op_cost(0);
 4332   format %{ %}
 4333   interface(CONST_INTER);
 4334 %}
 4335 
 4336 operand immI_255()
 4337 %{
 4338   predicate(n->get_int() == 255);
 4339   match(ConI);
 4340 
 4341   op_cost(0);
 4342   format %{ %}
 4343   interface(CONST_INTER);
 4344 %}
 4345 
 4346 operand immI_65535()
 4347 %{
 4348   predicate(n->get_int() == 65535);
 4349   match(ConI);
 4350 
 4351   op_cost(0);
 4352   format %{ %}
 4353   interface(CONST_INTER);
 4354 %}
 4355 
 4356 operand immI_positive()
 4357 %{
 4358   predicate(n->get_int() > 0);
 4359   match(ConI);
 4360 
 4361   op_cost(0);
 4362   format %{ %}
 4363   interface(CONST_INTER);
 4364 %}
 4365 
 4366 operand immL_255()
 4367 %{
 4368   predicate(n->get_long() == 255L);
 4369   match(ConL);
 4370 
 4371   op_cost(0);
 4372   format %{ %}
 4373   interface(CONST_INTER);
 4374 %}
 4375 
 4376 operand immL_65535()
 4377 %{
 4378   predicate(n->get_long() == 65535L);
 4379   match(ConL);
 4380 
 4381   op_cost(0);
 4382   format %{ %}
 4383   interface(CONST_INTER);
 4384 %}
 4385 
 4386 operand immL_4294967295()
 4387 %{
 4388   predicate(n->get_long() == 4294967295L);
 4389   match(ConL);
 4390 
 4391   op_cost(0);
 4392   format %{ %}
 4393   interface(CONST_INTER);
 4394 %}
 4395 
 4396 operand immL_bitmask()
 4397 %{
 4398   predicate((n->get_long() != 0)
 4399             && ((n->get_long() & 0xc000000000000000l) == 0)
 4400             && is_power_of_2(n->get_long() + 1));
 4401   match(ConL);
 4402 
 4403   op_cost(0);
 4404   format %{ %}
 4405   interface(CONST_INTER);
 4406 %}
 4407 
 4408 operand immI_bitmask()
 4409 %{
 4410   predicate((n->get_int() != 0)
 4411             && ((n->get_int() & 0xc0000000) == 0)
 4412             && is_power_of_2(n->get_int() + 1));
 4413   match(ConI);
 4414 
 4415   op_cost(0);
 4416   format %{ %}
 4417   interface(CONST_INTER);
 4418 %}
 4419 
 4420 operand immL_positive_bitmaskI()
 4421 %{
 4422   predicate((n->get_long() != 0)
 4423             && ((julong)n->get_long() < 0x80000000ULL)
 4424             && is_power_of_2(n->get_long() + 1));
 4425   match(ConL);
 4426 
 4427   op_cost(0);
 4428   format %{ %}
 4429   interface(CONST_INTER);
 4430 %}
 4431 
 4432 // Scale values for scaled offset addressing modes (up to long but not quad)
 4433 operand immIScale()
 4434 %{
 4435   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4436   match(ConI);
 4437 
 4438   op_cost(0);
 4439   format %{ %}
 4440   interface(CONST_INTER);
 4441 %}
 4442 
 4443 // 26 bit signed offset -- for pc-relative branches
 4444 operand immI26()
 4445 %{
 4446   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4447   match(ConI);
 4448 
 4449   op_cost(0);
 4450   format %{ %}
 4451   interface(CONST_INTER);
 4452 %}
 4453 
 4454 // 19 bit signed offset -- for pc-relative loads
 4455 operand immI19()
 4456 %{
 4457   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4458   match(ConI);
 4459 
 4460   op_cost(0);
 4461   format %{ %}
 4462   interface(CONST_INTER);
 4463 %}
 4464 
 4465 // 12 bit unsigned offset -- for base plus immediate loads
 4466 operand immIU12()
 4467 %{
 4468   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4469   match(ConI);
 4470 
 4471   op_cost(0);
 4472   format %{ %}
 4473   interface(CONST_INTER);
 4474 %}
 4475 
 4476 operand immLU12()
 4477 %{
 4478   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4479   match(ConL);
 4480 
 4481   op_cost(0);
 4482   format %{ %}
 4483   interface(CONST_INTER);
 4484 %}
 4485 
 4486 // Offset for scaled or unscaled immediate loads and stores
 4487 operand immIOffset()
 4488 %{
 4489   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4490   match(ConI);
 4491 
 4492   op_cost(0);
 4493   format %{ %}
 4494   interface(CONST_INTER);
 4495 %}
 4496 
 4497 operand immIOffset1()
 4498 %{
 4499   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4500   match(ConI);
 4501 
 4502   op_cost(0);
 4503   format %{ %}
 4504   interface(CONST_INTER);
 4505 %}
 4506 
 4507 operand immIOffset2()
 4508 %{
 4509   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4510   match(ConI);
 4511 
 4512   op_cost(0);
 4513   format %{ %}
 4514   interface(CONST_INTER);
 4515 %}
 4516 
 4517 operand immIOffset4()
 4518 %{
 4519   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4520   match(ConI);
 4521 
 4522   op_cost(0);
 4523   format %{ %}
 4524   interface(CONST_INTER);
 4525 %}
 4526 
 4527 operand immIOffset8()
 4528 %{
 4529   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4530   match(ConI);
 4531 
 4532   op_cost(0);
 4533   format %{ %}
 4534   interface(CONST_INTER);
 4535 %}
 4536 
 4537 operand immIOffset16()
 4538 %{
 4539   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4540   match(ConI);
 4541 
 4542   op_cost(0);
 4543   format %{ %}
 4544   interface(CONST_INTER);
 4545 %}
 4546 
 4547 operand immLoffset()
 4548 %{
 4549   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4550   match(ConL);
 4551 
 4552   op_cost(0);
 4553   format %{ %}
 4554   interface(CONST_INTER);
 4555 %}
 4556 
 4557 operand immLoffset1()
 4558 %{
 4559   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4560   match(ConL);
 4561 
 4562   op_cost(0);
 4563   format %{ %}
 4564   interface(CONST_INTER);
 4565 %}
 4566 
 4567 operand immLoffset2()
 4568 %{
 4569   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4570   match(ConL);
 4571 
 4572   op_cost(0);
 4573   format %{ %}
 4574   interface(CONST_INTER);
 4575 %}
 4576 
 4577 operand immLoffset4()
 4578 %{
 4579   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4580   match(ConL);
 4581 
 4582   op_cost(0);
 4583   format %{ %}
 4584   interface(CONST_INTER);
 4585 %}
 4586 
 4587 operand immLoffset8()
 4588 %{
 4589   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4590   match(ConL);
 4591 
 4592   op_cost(0);
 4593   format %{ %}
 4594   interface(CONST_INTER);
 4595 %}
 4596 
 4597 operand immLoffset16()
 4598 %{
 4599   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4600   match(ConL);
 4601 
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // 8 bit signed value.
 4608 operand immI8()
 4609 %{
 4610   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4611   match(ConI);
 4612 
 4613   op_cost(0);
 4614   format %{ %}
 4615   interface(CONST_INTER);
 4616 %}
 4617 
 4618 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4619 operand immI8_shift8()
 4620 %{
 4621   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4622             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4623   match(ConI);
 4624 
 4625   op_cost(0);
 4626   format %{ %}
 4627   interface(CONST_INTER);
 4628 %}
 4629 
 4630 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4631 operand immL8_shift8()
 4632 %{
 4633   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4634             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4635   match(ConL);
 4636 
 4637   op_cost(0);
 4638   format %{ %}
 4639   interface(CONST_INTER);
 4640 %}
 4641 
 4642 // 32 bit integer valid for add sub immediate
 4643 operand immIAddSub()
 4644 %{
 4645   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4646   match(ConI);
 4647   op_cost(0);
 4648   format %{ %}
 4649   interface(CONST_INTER);
 4650 %}
 4651 
 4652 // 32 bit unsigned integer valid for logical immediate
 4653 // TODO -- check this is right when e.g the mask is 0x80000000
 4654 operand immILog()
 4655 %{
 4656   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4657   match(ConI);
 4658 
 4659   op_cost(0);
 4660   format %{ %}
 4661   interface(CONST_INTER);
 4662 %}
 4663 
 4664 // Integer operands 64 bit
 4665 // 64 bit immediate
 4666 operand immL()
 4667 %{
 4668   match(ConL);
 4669 
 4670   op_cost(0);
 4671   format %{ %}
 4672   interface(CONST_INTER);
 4673 %}
 4674 
 4675 // 64 bit zero
 4676 operand immL0()
 4677 %{
 4678   predicate(n->get_long() == 0);
 4679   match(ConL);
 4680 
 4681   op_cost(0);
 4682   format %{ %}
 4683   interface(CONST_INTER);
 4684 %}
 4685 
 4686 // 64 bit unit increment
 4687 operand immL_1()
 4688 %{
 4689   predicate(n->get_long() == 1);
 4690   match(ConL);
 4691 
 4692   op_cost(0);
 4693   format %{ %}
 4694   interface(CONST_INTER);
 4695 %}
 4696 
 4697 // 64 bit unit decrement
 4698 operand immL_M1()
 4699 %{
 4700   predicate(n->get_long() == -1);
 4701   match(ConL);
 4702 
 4703   op_cost(0);
 4704   format %{ %}
 4705   interface(CONST_INTER);
 4706 %}
 4707 
 4708 // 32 bit offset of pc in thread anchor
 4709 
 4710 operand immL_pc_off()
 4711 %{
 4712   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4713                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4714   match(ConL);
 4715 
 4716   op_cost(0);
 4717   format %{ %}
 4718   interface(CONST_INTER);
 4719 %}
 4720 
 4721 // 64 bit integer valid for add sub immediate
 4722 operand immLAddSub()
 4723 %{
 4724   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4725   match(ConL);
 4726   op_cost(0);
 4727   format %{ %}
 4728   interface(CONST_INTER);
 4729 %}
 4730 
 4731 // 64 bit integer valid for logical immediate
 4732 operand immLLog()
 4733 %{
 4734   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4735   match(ConL);
 4736   op_cost(0);
 4737   format %{ %}
 4738   interface(CONST_INTER);
 4739 %}
 4740 
 4741 // Long Immediate: low 32-bit mask
 4742 operand immL_32bits()
 4743 %{
 4744   predicate(n->get_long() == 0xFFFFFFFFL);
 4745   match(ConL);
 4746   op_cost(0);
 4747   format %{ %}
 4748   interface(CONST_INTER);
 4749 %}
 4750 
 4751 // Pointer operands
 4752 // Pointer Immediate
 4753 operand immP()
 4754 %{
 4755   match(ConP);
 4756 
 4757   op_cost(0);
 4758   format %{ %}
 4759   interface(CONST_INTER);
 4760 %}
 4761 
 4762 // NULL Pointer Immediate
 4763 operand immP0()
 4764 %{
 4765   predicate(n->get_ptr() == 0);
 4766   match(ConP);
 4767 
 4768   op_cost(0);
 4769   format %{ %}
 4770   interface(CONST_INTER);
 4771 %}
 4772 
 4773 // Pointer Immediate One
 4774 // this is used in object initialization (initial object header)
 4775 operand immP_1()
 4776 %{
 4777   predicate(n->get_ptr() == 1);
 4778   match(ConP);
 4779 
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(CONST_INTER);
 4783 %}
 4784 
 4785 // Card Table Byte Map Base
 4786 operand immByteMapBase()
 4787 %{
 4788   // Get base of card map
 4789   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4790             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4791   match(ConP);
 4792 
 4793   op_cost(0);
 4794   format %{ %}
 4795   interface(CONST_INTER);
 4796 %}
 4797 
 4798 // Pointer Immediate Minus One
 4799 // this is used when we want to write the current PC to the thread anchor
 4800 operand immP_M1()
 4801 %{
 4802   predicate(n->get_ptr() == -1);
 4803   match(ConP);
 4804 
 4805   op_cost(0);
 4806   format %{ %}
 4807   interface(CONST_INTER);
 4808 %}
 4809 
 4810 // Pointer Immediate Minus Two
 4811 // this is used when we want to write the current PC to the thread anchor
 4812 operand immP_M2()
 4813 %{
 4814   predicate(n->get_ptr() == -2);
 4815   match(ConP);
 4816 
 4817   op_cost(0);
 4818   format %{ %}
 4819   interface(CONST_INTER);
 4820 %}
 4821 
 4822 // Float and Double operands
 4823 // Double Immediate
 4824 operand immD()
 4825 %{
 4826   match(ConD);
 4827   op_cost(0);
 4828   format %{ %}
 4829   interface(CONST_INTER);
 4830 %}
 4831 
 4832 // Double Immediate: +0.0d
 4833 operand immD0()
 4834 %{
 4835   predicate(jlong_cast(n->getd()) == 0);
 4836   match(ConD);
 4837 
 4838   op_cost(0);
 4839   format %{ %}
 4840   interface(CONST_INTER);
 4841 %}
 4842 
 4843 // constant 'double +0.0'.
 4844 operand immDPacked()
 4845 %{
 4846   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4847   match(ConD);
 4848   op_cost(0);
 4849   format %{ %}
 4850   interface(CONST_INTER);
 4851 %}
 4852 
 4853 // Float Immediate
 4854 operand immF()
 4855 %{
 4856   match(ConF);
 4857   op_cost(0);
 4858   format %{ %}
 4859   interface(CONST_INTER);
 4860 %}
 4861 
 4862 // Float Immediate: +0.0f.
 4863 operand immF0()
 4864 %{
 4865   predicate(jint_cast(n->getf()) == 0);
 4866   match(ConF);
 4867 
 4868   op_cost(0);
 4869   format %{ %}
 4870   interface(CONST_INTER);
 4871 %}
 4872 
 4873 //
 4874 operand immFPacked()
 4875 %{
 4876   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4877   match(ConF);
 4878   op_cost(0);
 4879   format %{ %}
 4880   interface(CONST_INTER);
 4881 %}
 4882 
 4883 // Narrow pointer operands
 4884 // Narrow Pointer Immediate
 4885 operand immN()
 4886 %{
 4887   match(ConN);
 4888 
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(CONST_INTER);
 4892 %}
 4893 
 4894 // Narrow NULL Pointer Immediate
 4895 operand immN0()
 4896 %{
 4897   predicate(n->get_narrowcon() == 0);
 4898   match(ConN);
 4899 
 4900   op_cost(0);
 4901   format %{ %}
 4902   interface(CONST_INTER);
 4903 %}
 4904 
 4905 operand immNKlass()
 4906 %{
 4907   match(ConNKlass);
 4908 
 4909   op_cost(0);
 4910   format %{ %}
 4911   interface(CONST_INTER);
 4912 %}
 4913 
 4914 // Integer 32 bit Register Operands
 4915 // Integer 32 bitRegister (excludes SP)
 4916 operand iRegI()
 4917 %{
 4918   constraint(ALLOC_IN_RC(any_reg32));
 4919   match(RegI);
 4920   match(iRegINoSp);
 4921   op_cost(0);
 4922   format %{ %}
 4923   interface(REG_INTER);
 4924 %}
 4925 
 4926 // Integer 32 bit Register not Special
 4927 operand iRegINoSp()
 4928 %{
 4929   constraint(ALLOC_IN_RC(no_special_reg32));
 4930   match(RegI);
 4931   op_cost(0);
 4932   format %{ %}
 4933   interface(REG_INTER);
 4934 %}
 4935 
 4936 // Integer 64 bit Register Operands
 4937 // Integer 64 bit Register (includes SP)
 4938 operand iRegL()
 4939 %{
 4940   constraint(ALLOC_IN_RC(any_reg));
 4941   match(RegL);
 4942   match(iRegLNoSp);
 4943   op_cost(0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 // Integer 64 bit Register not Special
 4949 operand iRegLNoSp()
 4950 %{
 4951   constraint(ALLOC_IN_RC(no_special_reg));
 4952   match(RegL);
 4953   match(iRegL_R0);
 4954   format %{ %}
 4955   interface(REG_INTER);
 4956 %}
 4957 
 4958 // Pointer Register Operands
 4959 // Pointer Register
 4960 operand iRegP()
 4961 %{
 4962   constraint(ALLOC_IN_RC(ptr_reg));
 4963   match(RegP);
 4964   match(iRegPNoSp);
 4965   match(iRegP_R0);
 4966   //match(iRegP_R2);
 4967   //match(iRegP_R4);
 4968   match(iRegP_R5);
 4969   match(thread_RegP);
 4970   op_cost(0);
 4971   format %{ %}
 4972   interface(REG_INTER);
 4973 %}
 4974 
 4975 // Pointer 64 bit Register not Special
 4976 operand iRegPNoSp()
 4977 %{
 4978   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4979   match(RegP);
 4980   // match(iRegP);
 4981   // match(iRegP_R0);
 4982   // match(iRegP_R2);
 4983   // match(iRegP_R4);
 4984   // match(iRegP_R5);
 4985   // match(thread_RegP);
 4986   op_cost(0);
 4987   format %{ %}
 4988   interface(REG_INTER);
 4989 %}
 4990 
 4991 // Pointer 64 bit Register R0 only
 4992 operand iRegP_R0()
 4993 %{
 4994   constraint(ALLOC_IN_RC(r0_reg));
 4995   match(RegP);
 4996   // match(iRegP);
 4997   match(iRegPNoSp);
 4998   op_cost(0);
 4999   format %{ %}
 5000   interface(REG_INTER);
 5001 %}
 5002 
 5003 // Pointer 64 bit Register R1 only
 5004 operand iRegP_R1()
 5005 %{
 5006   constraint(ALLOC_IN_RC(r1_reg));
 5007   match(RegP);
 5008   // match(iRegP);
 5009   match(iRegPNoSp);
 5010   op_cost(0);
 5011   format %{ %}
 5012   interface(REG_INTER);
 5013 %}
 5014 
 5015 // Pointer 64 bit Register R2 only
 5016 operand iRegP_R2()
 5017 %{
 5018   constraint(ALLOC_IN_RC(r2_reg));
 5019   match(RegP);
 5020   // match(iRegP);
 5021   match(iRegPNoSp);
 5022   op_cost(0);
 5023   format %{ %}
 5024   interface(REG_INTER);
 5025 %}
 5026 
 5027 // Pointer 64 bit Register R3 only
 5028 operand iRegP_R3()
 5029 %{
 5030   constraint(ALLOC_IN_RC(r3_reg));
 5031   match(RegP);
 5032   // match(iRegP);
 5033   match(iRegPNoSp);
 5034   op_cost(0);
 5035   format %{ %}
 5036   interface(REG_INTER);
 5037 %}
 5038 
 5039 // Pointer 64 bit Register R4 only
 5040 operand iRegP_R4()
 5041 %{
 5042   constraint(ALLOC_IN_RC(r4_reg));
 5043   match(RegP);
 5044   // match(iRegP);
 5045   match(iRegPNoSp);
 5046   op_cost(0);
 5047   format %{ %}
 5048   interface(REG_INTER);
 5049 %}
 5050 
 5051 // Pointer 64 bit Register R5 only
 5052 operand iRegP_R5()
 5053 %{
 5054   constraint(ALLOC_IN_RC(r5_reg));
 5055   match(RegP);
 5056   // match(iRegP);
 5057   match(iRegPNoSp);
 5058   op_cost(0);
 5059   format %{ %}
 5060   interface(REG_INTER);
 5061 %}
 5062 
 5063 // Pointer 64 bit Register R10 only
 5064 operand iRegP_R10()
 5065 %{
 5066   constraint(ALLOC_IN_RC(r10_reg));
 5067   match(RegP);
 5068   // match(iRegP);
 5069   match(iRegPNoSp);
 5070   op_cost(0);
 5071   format %{ %}
 5072   interface(REG_INTER);
 5073 %}
 5074 
 5075 // Long 64 bit Register R0 only
 5076 operand iRegL_R0()
 5077 %{
 5078   constraint(ALLOC_IN_RC(r0_reg));
 5079   match(RegL);
 5080   match(iRegLNoSp);
 5081   op_cost(0);
 5082   format %{ %}
 5083   interface(REG_INTER);
 5084 %}
 5085 
 5086 // Long 64 bit Register R2 only
 5087 operand iRegL_R2()
 5088 %{
 5089   constraint(ALLOC_IN_RC(r2_reg));
 5090   match(RegL);
 5091   match(iRegLNoSp);
 5092   op_cost(0);
 5093   format %{ %}
 5094   interface(REG_INTER);
 5095 %}
 5096 
 5097 // Long 64 bit Register R3 only
 5098 operand iRegL_R3()
 5099 %{
 5100   constraint(ALLOC_IN_RC(r3_reg));
 5101   match(RegL);
 5102   match(iRegLNoSp);
 5103   op_cost(0);
 5104   format %{ %}
 5105   interface(REG_INTER);
 5106 %}
 5107 
 5108 // Long 64 bit Register R11 only
 5109 operand iRegL_R11()
 5110 %{
 5111   constraint(ALLOC_IN_RC(r11_reg));
 5112   match(RegL);
 5113   match(iRegLNoSp);
 5114   op_cost(0);
 5115   format %{ %}
 5116   interface(REG_INTER);
 5117 %}
 5118 
 5119 // Pointer 64 bit Register FP only
 5120 operand iRegP_FP()
 5121 %{
 5122   constraint(ALLOC_IN_RC(fp_reg));
 5123   match(RegP);
 5124   // match(iRegP);
 5125   op_cost(0);
 5126   format %{ %}
 5127   interface(REG_INTER);
 5128 %}
 5129 
 5130 // Register R0 only
 5131 operand iRegI_R0()
 5132 %{
 5133   constraint(ALLOC_IN_RC(int_r0_reg));
 5134   match(RegI);
 5135   match(iRegINoSp);
 5136   op_cost(0);
 5137   format %{ %}
 5138   interface(REG_INTER);
 5139 %}
 5140 
 5141 // Register R2 only
 5142 operand iRegI_R2()
 5143 %{
 5144   constraint(ALLOC_IN_RC(int_r2_reg));
 5145   match(RegI);
 5146   match(iRegINoSp);
 5147   op_cost(0);
 5148   format %{ %}
 5149   interface(REG_INTER);
 5150 %}
 5151 
 5152 // Register R3 only
 5153 operand iRegI_R3()
 5154 %{
 5155   constraint(ALLOC_IN_RC(int_r3_reg));
 5156   match(RegI);
 5157   match(iRegINoSp);
 5158   op_cost(0);
 5159   format %{ %}
 5160   interface(REG_INTER);
 5161 %}
 5162 
 5163 
 5164 // Register R4 only
 5165 operand iRegI_R4()
 5166 %{
 5167   constraint(ALLOC_IN_RC(int_r4_reg));
 5168   match(RegI);
 5169   match(iRegINoSp);
 5170   op_cost(0);
 5171   format %{ %}
 5172   interface(REG_INTER);
 5173 %}
 5174 
 5175 
 5176 // Pointer Register Operands
 5177 // Narrow Pointer Register
 5178 operand iRegN()
 5179 %{
 5180   constraint(ALLOC_IN_RC(any_reg32));
 5181   match(RegN);
 5182   match(iRegNNoSp);
 5183   op_cost(0);
 5184   format %{ %}
 5185   interface(REG_INTER);
 5186 %}
 5187 
 5188 operand iRegN_R0()
 5189 %{
 5190   constraint(ALLOC_IN_RC(r0_reg));
 5191   match(iRegN);
 5192   op_cost(0);
 5193   format %{ %}
 5194   interface(REG_INTER);
 5195 %}
 5196 
 5197 operand iRegN_R2()
 5198 %{
 5199   constraint(ALLOC_IN_RC(r2_reg));
 5200   match(iRegN);
 5201   op_cost(0);
 5202   format %{ %}
 5203   interface(REG_INTER);
 5204 %}
 5205 
 5206 operand iRegN_R3()
 5207 %{
 5208   constraint(ALLOC_IN_RC(r3_reg));
 5209   match(iRegN);
 5210   op_cost(0);
 5211   format %{ %}
 5212   interface(REG_INTER);
 5213 %}
 5214 
 5215 // Integer 64 bit Register not Special
 5216 operand iRegNNoSp()
 5217 %{
 5218   constraint(ALLOC_IN_RC(no_special_reg32));
 5219   match(RegN);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 // heap base register -- used for encoding immN0
 5226 
 5227 operand iRegIHeapbase()
 5228 %{
 5229   constraint(ALLOC_IN_RC(heapbase_reg));
 5230   match(RegI);
 5231   op_cost(0);
 5232   format %{ %}
 5233   interface(REG_INTER);
 5234 %}
 5235 
 5236 // Float Register
 5237 // Float register operands
 5238 operand vRegF()
 5239 %{
 5240   constraint(ALLOC_IN_RC(float_reg));
 5241   match(RegF);
 5242 
 5243   op_cost(0);
 5244   format %{ %}
 5245   interface(REG_INTER);
 5246 %}
 5247 
 5248 // Double Register
 5249 // Double register operands
 5250 operand vRegD()
 5251 %{
 5252   constraint(ALLOC_IN_RC(double_reg));
 5253   match(RegD);
 5254 
 5255   op_cost(0);
 5256   format %{ %}
 5257   interface(REG_INTER);
 5258 %}
 5259 
 5260 // Generic vector class. This will be used for
 5261 // all vector operands, including NEON and SVE,
 5262 // but currently only used for SVE VecA.
 5263 operand vReg()
 5264 %{
 5265   constraint(ALLOC_IN_RC(vectora_reg));
 5266   match(VecA);
 5267   op_cost(0);
 5268   format %{ %}
 5269   interface(REG_INTER);
 5270 %}
 5271 
 5272 operand vecD()
 5273 %{
 5274   constraint(ALLOC_IN_RC(vectord_reg));
 5275   match(VecD);
 5276 
 5277   op_cost(0);
 5278   format %{ %}
 5279   interface(REG_INTER);
 5280 %}
 5281 
 5282 operand vecX()
 5283 %{
 5284   constraint(ALLOC_IN_RC(vectorx_reg));
 5285   match(VecX);
 5286 
 5287   op_cost(0);
 5288   format %{ %}
 5289   interface(REG_INTER);
 5290 %}
 5291 
 5292 operand vRegD_V0()
 5293 %{
 5294   constraint(ALLOC_IN_RC(v0_reg));
 5295   match(RegD);
 5296   op_cost(0);
 5297   format %{ %}
 5298   interface(REG_INTER);
 5299 %}
 5300 
 5301 operand vRegD_V1()
 5302 %{
 5303   constraint(ALLOC_IN_RC(v1_reg));
 5304   match(RegD);
 5305   op_cost(0);
 5306   format %{ %}
 5307   interface(REG_INTER);
 5308 %}
 5309 
 5310 operand vRegD_V2()
 5311 %{
 5312   constraint(ALLOC_IN_RC(v2_reg));
 5313   match(RegD);
 5314   op_cost(0);
 5315   format %{ %}
 5316   interface(REG_INTER);
 5317 %}
 5318 
 5319 operand vRegD_V3()
 5320 %{
 5321   constraint(ALLOC_IN_RC(v3_reg));
 5322   match(RegD);
 5323   op_cost(0);
 5324   format %{ %}
 5325   interface(REG_INTER);
 5326 %}
 5327 
 5328 operand vRegD_V4()
 5329 %{
 5330   constraint(ALLOC_IN_RC(v4_reg));
 5331   match(RegD);
 5332   op_cost(0);
 5333   format %{ %}
 5334   interface(REG_INTER);
 5335 %}
 5336 
 5337 operand vRegD_V5()
 5338 %{
 5339   constraint(ALLOC_IN_RC(v5_reg));
 5340   match(RegD);
 5341   op_cost(0);
 5342   format %{ %}
 5343   interface(REG_INTER);
 5344 %}
 5345 
 5346 operand vRegD_V6()
 5347 %{
 5348   constraint(ALLOC_IN_RC(v6_reg));
 5349   match(RegD);
 5350   op_cost(0);
 5351   format %{ %}
 5352   interface(REG_INTER);
 5353 %}
 5354 
 5355 operand vRegD_V7()
 5356 %{
 5357   constraint(ALLOC_IN_RC(v7_reg));
 5358   match(RegD);
 5359   op_cost(0);
 5360   format %{ %}
 5361   interface(REG_INTER);
 5362 %}
 5363 
 5364 operand vRegD_V8()
 5365 %{
 5366   constraint(ALLOC_IN_RC(v8_reg));
 5367   match(RegD);
 5368   op_cost(0);
 5369   format %{ %}
 5370   interface(REG_INTER);
 5371 %}
 5372 
 5373 operand vRegD_V9()
 5374 %{
 5375   constraint(ALLOC_IN_RC(v9_reg));
 5376   match(RegD);
 5377   op_cost(0);
 5378   format %{ %}
 5379   interface(REG_INTER);
 5380 %}
 5381 
 5382 operand vRegD_V10()
 5383 %{
 5384   constraint(ALLOC_IN_RC(v10_reg));
 5385   match(RegD);
 5386   op_cost(0);
 5387   format %{ %}
 5388   interface(REG_INTER);
 5389 %}
 5390 
 5391 operand vRegD_V11()
 5392 %{
 5393   constraint(ALLOC_IN_RC(v11_reg));
 5394   match(RegD);
 5395   op_cost(0);
 5396   format %{ %}
 5397   interface(REG_INTER);
 5398 %}
 5399 
 5400 operand vRegD_V12()
 5401 %{
 5402   constraint(ALLOC_IN_RC(v12_reg));
 5403   match(RegD);
 5404   op_cost(0);
 5405   format %{ %}
 5406   interface(REG_INTER);
 5407 %}
 5408 
 5409 operand vRegD_V13()
 5410 %{
 5411   constraint(ALLOC_IN_RC(v13_reg));
 5412   match(RegD);
 5413   op_cost(0);
 5414   format %{ %}
 5415   interface(REG_INTER);
 5416 %}
 5417 
 5418 operand vRegD_V14()
 5419 %{
 5420   constraint(ALLOC_IN_RC(v14_reg));
 5421   match(RegD);
 5422   op_cost(0);
 5423   format %{ %}
 5424   interface(REG_INTER);
 5425 %}
 5426 
 5427 operand vRegD_V15()
 5428 %{
 5429   constraint(ALLOC_IN_RC(v15_reg));
 5430   match(RegD);
 5431   op_cost(0);
 5432   format %{ %}
 5433   interface(REG_INTER);
 5434 %}
 5435 
 5436 operand vRegD_V16()
 5437 %{
 5438   constraint(ALLOC_IN_RC(v16_reg));
 5439   match(RegD);
 5440   op_cost(0);
 5441   format %{ %}
 5442   interface(REG_INTER);
 5443 %}
 5444 
 5445 operand vRegD_V17()
 5446 %{
 5447   constraint(ALLOC_IN_RC(v17_reg));
 5448   match(RegD);
 5449   op_cost(0);
 5450   format %{ %}
 5451   interface(REG_INTER);
 5452 %}
 5453 
 5454 operand vRegD_V18()
 5455 %{
 5456   constraint(ALLOC_IN_RC(v18_reg));
 5457   match(RegD);
 5458   op_cost(0);
 5459   format %{ %}
 5460   interface(REG_INTER);
 5461 %}
 5462 
 5463 operand vRegD_V19()
 5464 %{
 5465   constraint(ALLOC_IN_RC(v19_reg));
 5466   match(RegD);
 5467   op_cost(0);
 5468   format %{ %}
 5469   interface(REG_INTER);
 5470 %}
 5471 
 5472 operand vRegD_V20()
 5473 %{
 5474   constraint(ALLOC_IN_RC(v20_reg));
 5475   match(RegD);
 5476   op_cost(0);
 5477   format %{ %}
 5478   interface(REG_INTER);
 5479 %}
 5480 
 5481 operand vRegD_V21()
 5482 %{
 5483   constraint(ALLOC_IN_RC(v21_reg));
 5484   match(RegD);
 5485   op_cost(0);
 5486   format %{ %}
 5487   interface(REG_INTER);
 5488 %}
 5489 
 5490 operand vRegD_V22()
 5491 %{
 5492   constraint(ALLOC_IN_RC(v22_reg));
 5493   match(RegD);
 5494   op_cost(0);
 5495   format %{ %}
 5496   interface(REG_INTER);
 5497 %}
 5498 
 5499 operand vRegD_V23()
 5500 %{
 5501   constraint(ALLOC_IN_RC(v23_reg));
 5502   match(RegD);
 5503   op_cost(0);
 5504   format %{ %}
 5505   interface(REG_INTER);
 5506 %}
 5507 
 5508 operand vRegD_V24()
 5509 %{
 5510   constraint(ALLOC_IN_RC(v24_reg));
 5511   match(RegD);
 5512   op_cost(0);
 5513   format %{ %}
 5514   interface(REG_INTER);
 5515 %}
 5516 
 5517 operand vRegD_V25()
 5518 %{
 5519   constraint(ALLOC_IN_RC(v25_reg));
 5520   match(RegD);
 5521   op_cost(0);
 5522   format %{ %}
 5523   interface(REG_INTER);
 5524 %}
 5525 
 5526 operand vRegD_V26()
 5527 %{
 5528   constraint(ALLOC_IN_RC(v26_reg));
 5529   match(RegD);
 5530   op_cost(0);
 5531   format %{ %}
 5532   interface(REG_INTER);
 5533 %}
 5534 
 5535 operand vRegD_V27()
 5536 %{
 5537   constraint(ALLOC_IN_RC(v27_reg));
 5538   match(RegD);
 5539   op_cost(0);
 5540   format %{ %}
 5541   interface(REG_INTER);
 5542 %}
 5543 
 5544 operand vRegD_V28()
 5545 %{
 5546   constraint(ALLOC_IN_RC(v28_reg));
 5547   match(RegD);
 5548   op_cost(0);
 5549   format %{ %}
 5550   interface(REG_INTER);
 5551 %}
 5552 
 5553 operand vRegD_V29()
 5554 %{
 5555   constraint(ALLOC_IN_RC(v29_reg));
 5556   match(RegD);
 5557   op_cost(0);
 5558   format %{ %}
 5559   interface(REG_INTER);
 5560 %}
 5561 
 5562 operand vRegD_V30()
 5563 %{
 5564   constraint(ALLOC_IN_RC(v30_reg));
 5565   match(RegD);
 5566   op_cost(0);
 5567   format %{ %}
 5568   interface(REG_INTER);
 5569 %}
 5570 
 5571 operand vRegD_V31()
 5572 %{
 5573   constraint(ALLOC_IN_RC(v31_reg));
 5574   match(RegD);
 5575   op_cost(0);
 5576   format %{ %}
 5577   interface(REG_INTER);
 5578 %}
 5579 
 5580 operand pRegGov()
 5581 %{
 5582   constraint(ALLOC_IN_RC(gov_pr));
 5583   match(RegVectMask);
 5584   op_cost(0);
 5585   format %{ %}
 5586   interface(REG_INTER);
 5587 %}
 5588 
 5589 // Flags register, used as output of signed compare instructions
 5590 
 5591 // note that on AArch64 we also use this register as the output for
 5592 // for floating point compare instructions (CmpF CmpD). this ensures
 5593 // that ordered inequality tests use GT, GE, LT or LE none of which
 5594 // pass through cases where the result is unordered i.e. one or both
 5595 // inputs to the compare is a NaN. this means that the ideal code can
 5596 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5597 // (where the comparison should always fail). EQ and NE tests are
 5598 // always generated in ideal code so that unordered folds into the NE
 5599 // case, matching the behaviour of AArch64 NE.
 5600 //
 5601 // This differs from x86 where the outputs of FP compares use a
 5602 // special FP flags registers and where compares based on this
 5603 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5604 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5605 // to explicitly handle the unordered case in branches. x86 also has
 5606 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5607 
 5608 operand rFlagsReg()
 5609 %{
 5610   constraint(ALLOC_IN_RC(int_flags));
 5611   match(RegFlags);
 5612 
 5613   op_cost(0);
 5614   format %{ "RFLAGS" %}
 5615   interface(REG_INTER);
 5616 %}
 5617 
 5618 // Flags register, used as output of unsigned compare instructions
 5619 operand rFlagsRegU()
 5620 %{
 5621   constraint(ALLOC_IN_RC(int_flags));
 5622   match(RegFlags);
 5623 
 5624   op_cost(0);
 5625   format %{ "RFLAGSU" %}
 5626   interface(REG_INTER);
 5627 %}
 5628 
 5629 // Special Registers
 5630 
 5631 // Method Register
 5632 operand inline_cache_RegP(iRegP reg)
 5633 %{
 5634   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5635   match(reg);
 5636   match(iRegPNoSp);
 5637   op_cost(0);
 5638   format %{ %}
 5639   interface(REG_INTER);
 5640 %}
 5641 
 5642 // Thread Register
 5643 operand thread_RegP(iRegP reg)
 5644 %{
 5645   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5646   match(reg);
 5647   op_cost(0);
 5648   format %{ %}
 5649   interface(REG_INTER);
 5650 %}
 5651 
 5652 operand lr_RegP(iRegP reg)
 5653 %{
 5654   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5655   match(reg);
 5656   op_cost(0);
 5657   format %{ %}
 5658   interface(REG_INTER);
 5659 %}
 5660 
 5661 //----------Memory Operands----------------------------------------------------
 5662 
 5663 operand indirect(iRegP reg)
 5664 %{
 5665   constraint(ALLOC_IN_RC(ptr_reg));
 5666   match(reg);
 5667   op_cost(0);
 5668   format %{ "[$reg]" %}
 5669   interface(MEMORY_INTER) %{
 5670     base($reg);
 5671     index(0xffffffff);
 5672     scale(0x0);
 5673     disp(0x0);
 5674   %}
 5675 %}
 5676 
 5677 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5678 %{
 5679   constraint(ALLOC_IN_RC(ptr_reg));
 5680   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5681   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5682   op_cost(0);
 5683   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5684   interface(MEMORY_INTER) %{
 5685     base($reg);
 5686     index($ireg);
 5687     scale($scale);
 5688     disp(0x0);
 5689   %}
 5690 %}
 5691 
 5692 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5693 %{
 5694   constraint(ALLOC_IN_RC(ptr_reg));
 5695   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5696   match(AddP reg (LShiftL lreg scale));
 5697   op_cost(0);
 5698   format %{ "$reg, $lreg lsl($scale)" %}
 5699   interface(MEMORY_INTER) %{
 5700     base($reg);
 5701     index($lreg);
 5702     scale($scale);
 5703     disp(0x0);
 5704   %}
 5705 %}
 5706 
 5707 operand indIndexI2L(iRegP reg, iRegI ireg)
 5708 %{
 5709   constraint(ALLOC_IN_RC(ptr_reg));
 5710   match(AddP reg (ConvI2L ireg));
 5711   op_cost(0);
 5712   format %{ "$reg, $ireg, 0, I2L" %}
 5713   interface(MEMORY_INTER) %{
 5714     base($reg);
 5715     index($ireg);
 5716     scale(0x0);
 5717     disp(0x0);
 5718   %}
 5719 %}
 5720 
 5721 operand indIndex(iRegP reg, iRegL lreg)
 5722 %{
 5723   constraint(ALLOC_IN_RC(ptr_reg));
 5724   match(AddP reg lreg);
 5725   op_cost(0);
 5726   format %{ "$reg, $lreg" %}
 5727   interface(MEMORY_INTER) %{
 5728     base($reg);
 5729     index($lreg);
 5730     scale(0x0);
 5731     disp(0x0);
 5732   %}
 5733 %}
 5734 
 5735 operand indOffI(iRegP reg, immIOffset off)
 5736 %{
 5737   constraint(ALLOC_IN_RC(ptr_reg));
 5738   match(AddP reg off);
 5739   op_cost(0);
 5740   format %{ "[$reg, $off]" %}
 5741   interface(MEMORY_INTER) %{
 5742     base($reg);
 5743     index(0xffffffff);
 5744     scale(0x0);
 5745     disp($off);
 5746   %}
 5747 %}
 5748 
 5749 operand indOffI1(iRegP reg, immIOffset1 off)
 5750 %{
 5751   constraint(ALLOC_IN_RC(ptr_reg));
 5752   match(AddP reg off);
 5753   op_cost(0);
 5754   format %{ "[$reg, $off]" %}
 5755   interface(MEMORY_INTER) %{
 5756     base($reg);
 5757     index(0xffffffff);
 5758     scale(0x0);
 5759     disp($off);
 5760   %}
 5761 %}
 5762 
 5763 operand indOffI2(iRegP reg, immIOffset2 off)
 5764 %{
 5765   constraint(ALLOC_IN_RC(ptr_reg));
 5766   match(AddP reg off);
 5767   op_cost(0);
 5768   format %{ "[$reg, $off]" %}
 5769   interface(MEMORY_INTER) %{
 5770     base($reg);
 5771     index(0xffffffff);
 5772     scale(0x0);
 5773     disp($off);
 5774   %}
 5775 %}
 5776 
 5777 operand indOffI4(iRegP reg, immIOffset4 off)
 5778 %{
 5779   constraint(ALLOC_IN_RC(ptr_reg));
 5780   match(AddP reg off);
 5781   op_cost(0);
 5782   format %{ "[$reg, $off]" %}
 5783   interface(MEMORY_INTER) %{
 5784     base($reg);
 5785     index(0xffffffff);
 5786     scale(0x0);
 5787     disp($off);
 5788   %}
 5789 %}
 5790 
 5791 operand indOffI8(iRegP reg, immIOffset8 off)
 5792 %{
 5793   constraint(ALLOC_IN_RC(ptr_reg));
 5794   match(AddP reg off);
 5795   op_cost(0);
 5796   format %{ "[$reg, $off]" %}
 5797   interface(MEMORY_INTER) %{
 5798     base($reg);
 5799     index(0xffffffff);
 5800     scale(0x0);
 5801     disp($off);
 5802   %}
 5803 %}
 5804 
 5805 operand indOffI16(iRegP reg, immIOffset16 off)
 5806 %{
 5807   constraint(ALLOC_IN_RC(ptr_reg));
 5808   match(AddP reg off);
 5809   op_cost(0);
 5810   format %{ "[$reg, $off]" %}
 5811   interface(MEMORY_INTER) %{
 5812     base($reg);
 5813     index(0xffffffff);
 5814     scale(0x0);
 5815     disp($off);
 5816   %}
 5817 %}
 5818 
 5819 operand indOffL(iRegP reg, immLoffset off)
 5820 %{
 5821   constraint(ALLOC_IN_RC(ptr_reg));
 5822   match(AddP reg off);
 5823   op_cost(0);
 5824   format %{ "[$reg, $off]" %}
 5825   interface(MEMORY_INTER) %{
 5826     base($reg);
 5827     index(0xffffffff);
 5828     scale(0x0);
 5829     disp($off);
 5830   %}
 5831 %}
 5832 
 5833 operand indOffL1(iRegP reg, immLoffset1 off)
 5834 %{
 5835   constraint(ALLOC_IN_RC(ptr_reg));
 5836   match(AddP reg off);
 5837   op_cost(0);
 5838   format %{ "[$reg, $off]" %}
 5839   interface(MEMORY_INTER) %{
 5840     base($reg);
 5841     index(0xffffffff);
 5842     scale(0x0);
 5843     disp($off);
 5844   %}
 5845 %}
 5846 
 5847 operand indOffL2(iRegP reg, immLoffset2 off)
 5848 %{
 5849   constraint(ALLOC_IN_RC(ptr_reg));
 5850   match(AddP reg off);
 5851   op_cost(0);
 5852   format %{ "[$reg, $off]" %}
 5853   interface(MEMORY_INTER) %{
 5854     base($reg);
 5855     index(0xffffffff);
 5856     scale(0x0);
 5857     disp($off);
 5858   %}
 5859 %}
 5860 
 5861 operand indOffL4(iRegP reg, immLoffset4 off)
 5862 %{
 5863   constraint(ALLOC_IN_RC(ptr_reg));
 5864   match(AddP reg off);
 5865   op_cost(0);
 5866   format %{ "[$reg, $off]" %}
 5867   interface(MEMORY_INTER) %{
 5868     base($reg);
 5869     index(0xffffffff);
 5870     scale(0x0);
 5871     disp($off);
 5872   %}
 5873 %}
 5874 
 5875 operand indOffL8(iRegP reg, immLoffset8 off)
 5876 %{
 5877   constraint(ALLOC_IN_RC(ptr_reg));
 5878   match(AddP reg off);
 5879   op_cost(0);
 5880   format %{ "[$reg, $off]" %}
 5881   interface(MEMORY_INTER) %{
 5882     base($reg);
 5883     index(0xffffffff);
 5884     scale(0x0);
 5885     disp($off);
 5886   %}
 5887 %}
 5888 
 5889 operand indOffL16(iRegP reg, immLoffset16 off)
 5890 %{
 5891   constraint(ALLOC_IN_RC(ptr_reg));
 5892   match(AddP reg off);
 5893   op_cost(0);
 5894   format %{ "[$reg, $off]" %}
 5895   interface(MEMORY_INTER) %{
 5896     base($reg);
 5897     index(0xffffffff);
 5898     scale(0x0);
 5899     disp($off);
 5900   %}
 5901 %}
 5902 
 5903 operand indirectN(iRegN reg)
 5904 %{
 5905   predicate(CompressedOops::shift() == 0);
 5906   constraint(ALLOC_IN_RC(ptr_reg));
 5907   match(DecodeN reg);
 5908   op_cost(0);
 5909   format %{ "[$reg]\t# narrow" %}
 5910   interface(MEMORY_INTER) %{
 5911     base($reg);
 5912     index(0xffffffff);
 5913     scale(0x0);
 5914     disp(0x0);
 5915   %}
 5916 %}
 5917 
 5918 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5919 %{
 5920   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5921   constraint(ALLOC_IN_RC(ptr_reg));
 5922   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5923   op_cost(0);
 5924   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5925   interface(MEMORY_INTER) %{
 5926     base($reg);
 5927     index($ireg);
 5928     scale($scale);
 5929     disp(0x0);
 5930   %}
 5931 %}
 5932 
 5933 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5934 %{
 5935   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5936   constraint(ALLOC_IN_RC(ptr_reg));
 5937   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5938   op_cost(0);
 5939   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5940   interface(MEMORY_INTER) %{
 5941     base($reg);
 5942     index($lreg);
 5943     scale($scale);
 5944     disp(0x0);
 5945   %}
 5946 %}
 5947 
 5948 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5949 %{
 5950   predicate(CompressedOops::shift() == 0);
 5951   constraint(ALLOC_IN_RC(ptr_reg));
 5952   match(AddP (DecodeN reg) (ConvI2L ireg));
 5953   op_cost(0);
 5954   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5955   interface(MEMORY_INTER) %{
 5956     base($reg);
 5957     index($ireg);
 5958     scale(0x0);
 5959     disp(0x0);
 5960   %}
 5961 %}
 5962 
 5963 operand indIndexN(iRegN reg, iRegL lreg)
 5964 %{
 5965   predicate(CompressedOops::shift() == 0);
 5966   constraint(ALLOC_IN_RC(ptr_reg));
 5967   match(AddP (DecodeN reg) lreg);
 5968   op_cost(0);
 5969   format %{ "$reg, $lreg\t# narrow" %}
 5970   interface(MEMORY_INTER) %{
 5971     base($reg);
 5972     index($lreg);
 5973     scale(0x0);
 5974     disp(0x0);
 5975   %}
 5976 %}
 5977 
 5978 operand indOffIN(iRegN reg, immIOffset off)
 5979 %{
 5980   predicate(CompressedOops::shift() == 0);
 5981   constraint(ALLOC_IN_RC(ptr_reg));
 5982   match(AddP (DecodeN reg) off);
 5983   op_cost(0);
 5984   format %{ "[$reg, $off]\t# narrow" %}
 5985   interface(MEMORY_INTER) %{
 5986     base($reg);
 5987     index(0xffffffff);
 5988     scale(0x0);
 5989     disp($off);
 5990   %}
 5991 %}
 5992 
 5993 operand indOffLN(iRegN reg, immLoffset off)
 5994 %{
 5995   predicate(CompressedOops::shift() == 0);
 5996   constraint(ALLOC_IN_RC(ptr_reg));
 5997   match(AddP (DecodeN reg) off);
 5998   op_cost(0);
 5999   format %{ "[$reg, $off]\t# narrow" %}
 6000   interface(MEMORY_INTER) %{
 6001     base($reg);
 6002     index(0xffffffff);
 6003     scale(0x0);
 6004     disp($off);
 6005   %}
 6006 %}
 6007 
 6008 
 6009 
 6010 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6011 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6012 %{
 6013   constraint(ALLOC_IN_RC(ptr_reg));
 6014   match(AddP reg off);
 6015   op_cost(0);
 6016   format %{ "[$reg, $off]" %}
 6017   interface(MEMORY_INTER) %{
 6018     base($reg);
 6019     index(0xffffffff);
 6020     scale(0x0);
 6021     disp($off);
 6022   %}
 6023 %}
 6024 
 6025 //----------Special Memory Operands--------------------------------------------
 6026 // Stack Slot Operand - This operand is used for loading and storing temporary
 6027 //                      values on the stack where a match requires a value to
 6028 //                      flow through memory.
 6029 operand stackSlotP(sRegP reg)
 6030 %{
 6031   constraint(ALLOC_IN_RC(stack_slots));
 6032   op_cost(100);
 6033   // No match rule because this operand is only generated in matching
 6034   // match(RegP);
 6035   format %{ "[$reg]" %}
 6036   interface(MEMORY_INTER) %{
 6037     base(0x1e);  // RSP
 6038     index(0x0);  // No Index
 6039     scale(0x0);  // No Scale
 6040     disp($reg);  // Stack Offset
 6041   %}
 6042 %}
 6043 
 6044 operand stackSlotI(sRegI reg)
 6045 %{
 6046   constraint(ALLOC_IN_RC(stack_slots));
 6047   // No match rule because this operand is only generated in matching
 6048   // match(RegI);
 6049   format %{ "[$reg]" %}
 6050   interface(MEMORY_INTER) %{
 6051     base(0x1e);  // RSP
 6052     index(0x0);  // No Index
 6053     scale(0x0);  // No Scale
 6054     disp($reg);  // Stack Offset
 6055   %}
 6056 %}
 6057 
 6058 operand stackSlotF(sRegF reg)
 6059 %{
 6060   constraint(ALLOC_IN_RC(stack_slots));
 6061   // No match rule because this operand is only generated in matching
 6062   // match(RegF);
 6063   format %{ "[$reg]" %}
 6064   interface(MEMORY_INTER) %{
 6065     base(0x1e);  // RSP
 6066     index(0x0);  // No Index
 6067     scale(0x0);  // No Scale
 6068     disp($reg);  // Stack Offset
 6069   %}
 6070 %}
 6071 
 6072 operand stackSlotD(sRegD reg)
 6073 %{
 6074   constraint(ALLOC_IN_RC(stack_slots));
 6075   // No match rule because this operand is only generated in matching
 6076   // match(RegD);
 6077   format %{ "[$reg]" %}
 6078   interface(MEMORY_INTER) %{
 6079     base(0x1e);  // RSP
 6080     index(0x0);  // No Index
 6081     scale(0x0);  // No Scale
 6082     disp($reg);  // Stack Offset
 6083   %}
 6084 %}
 6085 
 6086 operand stackSlotL(sRegL reg)
 6087 %{
 6088   constraint(ALLOC_IN_RC(stack_slots));
 6089   // No match rule because this operand is only generated in matching
 6090   // match(RegL);
 6091   format %{ "[$reg]" %}
 6092   interface(MEMORY_INTER) %{
 6093     base(0x1e);  // RSP
 6094     index(0x0);  // No Index
 6095     scale(0x0);  // No Scale
 6096     disp($reg);  // Stack Offset
 6097   %}
 6098 %}
 6099 
 6100 // Operands for expressing Control Flow
 6101 // NOTE: Label is a predefined operand which should not be redefined in
 6102 //       the AD file. It is generically handled within the ADLC.
 6103 
 6104 //----------Conditional Branch Operands----------------------------------------
 6105 // Comparison Op  - This is the operation of the comparison, and is limited to
 6106 //                  the following set of codes:
 6107 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6108 //
 6109 // Other attributes of the comparison, such as unsignedness, are specified
 6110 // by the comparison instruction that sets a condition code flags register.
 6111 // That result is represented by a flags operand whose subtype is appropriate
 6112 // to the unsignedness (etc.) of the comparison.
 6113 //
 6114 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6115 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6116 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6117 
 6118 // used for signed integral comparisons and fp comparisons
 6119 
 6120 operand cmpOp()
 6121 %{
 6122   match(Bool);
 6123 
 6124   format %{ "" %}
 6125   interface(COND_INTER) %{
 6126     equal(0x0, "eq");
 6127     not_equal(0x1, "ne");
 6128     less(0xb, "lt");
 6129     greater_equal(0xa, "ge");
 6130     less_equal(0xd, "le");
 6131     greater(0xc, "gt");
 6132     overflow(0x6, "vs");
 6133     no_overflow(0x7, "vc");
 6134   %}
 6135 %}
 6136 
 6137 // used for unsigned integral comparisons
 6138 
 6139 operand cmpOpU()
 6140 %{
 6141   match(Bool);
 6142 
 6143   format %{ "" %}
 6144   interface(COND_INTER) %{
 6145     equal(0x0, "eq");
 6146     not_equal(0x1, "ne");
 6147     less(0x3, "lo");
 6148     greater_equal(0x2, "hs");
 6149     less_equal(0x9, "ls");
 6150     greater(0x8, "hi");
 6151     overflow(0x6, "vs");
 6152     no_overflow(0x7, "vc");
 6153   %}
 6154 %}
 6155 
 6156 // used for certain integral comparisons which can be
 6157 // converted to cbxx or tbxx instructions
 6158 
 6159 operand cmpOpEqNe()
 6160 %{
 6161   match(Bool);
 6162   op_cost(0);
 6163   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6164             || n->as_Bool()->_test._test == BoolTest::eq);
 6165 
 6166   format %{ "" %}
 6167   interface(COND_INTER) %{
 6168     equal(0x0, "eq");
 6169     not_equal(0x1, "ne");
 6170     less(0xb, "lt");
 6171     greater_equal(0xa, "ge");
 6172     less_equal(0xd, "le");
 6173     greater(0xc, "gt");
 6174     overflow(0x6, "vs");
 6175     no_overflow(0x7, "vc");
 6176   %}
 6177 %}
 6178 
 6179 // used for certain integral comparisons which can be
 6180 // converted to cbxx or tbxx instructions
 6181 
 6182 operand cmpOpLtGe()
 6183 %{
 6184   match(Bool);
 6185   op_cost(0);
 6186 
 6187   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6188             || n->as_Bool()->_test._test == BoolTest::ge);
 6189 
 6190   format %{ "" %}
 6191   interface(COND_INTER) %{
 6192     equal(0x0, "eq");
 6193     not_equal(0x1, "ne");
 6194     less(0xb, "lt");
 6195     greater_equal(0xa, "ge");
 6196     less_equal(0xd, "le");
 6197     greater(0xc, "gt");
 6198     overflow(0x6, "vs");
 6199     no_overflow(0x7, "vc");
 6200   %}
 6201 %}
 6202 
 6203 // used for certain unsigned integral comparisons which can be
 6204 // converted to cbxx or tbxx instructions
 6205 
 6206 operand cmpOpUEqNeLtGe()
 6207 %{
 6208   match(Bool);
 6209   op_cost(0);
 6210 
 6211   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6212             || n->as_Bool()->_test._test == BoolTest::ne
 6213             || n->as_Bool()->_test._test == BoolTest::lt
 6214             || n->as_Bool()->_test._test == BoolTest::ge);
 6215 
 6216   format %{ "" %}
 6217   interface(COND_INTER) %{
 6218     equal(0x0, "eq");
 6219     not_equal(0x1, "ne");
 6220     less(0xb, "lt");
 6221     greater_equal(0xa, "ge");
 6222     less_equal(0xd, "le");
 6223     greater(0xc, "gt");
 6224     overflow(0x6, "vs");
 6225     no_overflow(0x7, "vc");
 6226   %}
 6227 %}
 6228 
 6229 // Special operand allowing long args to int ops to be truncated for free
 6230 
 6231 operand iRegL2I(iRegL reg) %{
 6232 
 6233   op_cost(0);
 6234 
 6235   match(ConvL2I reg);
 6236 
 6237   format %{ "l2i($reg)" %}
 6238 
 6239   interface(REG_INTER)
 6240 %}
 6241 
 6242 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6243 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6244 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6245 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6246 
 6247 //----------OPERAND CLASSES----------------------------------------------------
 6248 // Operand Classes are groups of operands that are used as to simplify
 6249 // instruction definitions by not requiring the AD writer to specify
 6250 // separate instructions for every form of operand when the
 6251 // instruction accepts multiple operand types with the same basic
 6252 // encoding and format. The classic case of this is memory operands.
 6253 
 6254 // memory is used to define read/write location for load/store
 6255 // instruction defs. we can turn a memory op into an Address
 6256 
 6257 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6258                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6259 
 6260 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6261                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6262 
 6263 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6264                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6265 
 6266 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6267                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6268 
 6269 // All of the memory operands. For the pipeline description.
 6270 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6271                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6272                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6273 
 6274 
 6275 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6276 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6277 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6278 // can be elided because the 32-bit instruction will just employ the
 6279 // lower 32 bits anyway.
 6280 //
 6281 // n.b. this does not elide all L2I conversions. if the truncated
 6282 // value is consumed by more than one operation then the ConvL2I
 6283 // cannot be bundled into the consuming nodes so an l2i gets planted
 6284 // (actually a movw $dst $src) and the downstream instructions consume
 6285 // the result of the l2i as an iRegI input. That's a shame since the
 6286 // movw is actually redundant but its not too costly.
 6287 
 6288 opclass iRegIorL2I(iRegI, iRegL2I);
 6289 
 6290 //----------PIPELINE-----------------------------------------------------------
 6291 // Rules which define the behavior of the target architectures pipeline.
 6292 
 6293 // For specific pipelines, eg A53, define the stages of that pipeline
 6294 //pipe_desc(ISS, EX1, EX2, WR);
 6295 #define ISS S0
 6296 #define EX1 S1
 6297 #define EX2 S2
 6298 #define WR  S3
 6299 
 6300 // Integer ALU reg operation
 6301 pipeline %{
 6302 
 6303 attributes %{
 6304   // ARM instructions are of fixed length
 6305   fixed_size_instructions;        // Fixed size instructions TODO does
 6306   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6307   // ARM instructions come in 32-bit word units
 6308   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6309   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6310   instruction_fetch_units = 1;       // of 64 bytes
 6311 
 6312   // List of nop instructions
 6313   nops( MachNop );
 6314 %}
 6315 
 6316 // We don't use an actual pipeline model so don't care about resources
 6317 // or description. we do use pipeline classes to introduce fixed
 6318 // latencies
 6319 
 6320 //----------RESOURCES----------------------------------------------------------
 6321 // Resources are the functional units available to the machine
 6322 
 6323 resources( INS0, INS1, INS01 = INS0 | INS1,
 6324            ALU0, ALU1, ALU = ALU0 | ALU1,
 6325            MAC,
 6326            DIV,
 6327            BRANCH,
 6328            LDST,
 6329            NEON_FP);
 6330 
 6331 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6332 // Pipeline Description specifies the stages in the machine's pipeline
 6333 
 6334 // Define the pipeline as a generic 6 stage pipeline
 6335 pipe_desc(S0, S1, S2, S3, S4, S5);
 6336 
 6337 //----------PIPELINE CLASSES---------------------------------------------------
 6338 // Pipeline Classes describe the stages in which input and output are
 6339 // referenced by the hardware pipeline.
 6340 
 6341 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6342 %{
 6343   single_instruction;
 6344   src1   : S1(read);
 6345   src2   : S2(read);
 6346   dst    : S5(write);
 6347   INS01  : ISS;
 6348   NEON_FP : S5;
 6349 %}
 6350 
 6351 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6352 %{
 6353   single_instruction;
 6354   src1   : S1(read);
 6355   src2   : S2(read);
 6356   dst    : S5(write);
 6357   INS01  : ISS;
 6358   NEON_FP : S5;
 6359 %}
 6360 
 6361 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6362 %{
 6363   single_instruction;
 6364   src    : S1(read);
 6365   dst    : S5(write);
 6366   INS01  : ISS;
 6367   NEON_FP : S5;
 6368 %}
 6369 
 6370 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6371 %{
 6372   single_instruction;
 6373   src    : S1(read);
 6374   dst    : S5(write);
 6375   INS01  : ISS;
 6376   NEON_FP : S5;
 6377 %}
 6378 
 6379 pipe_class fp_d2f(vRegF dst, vRegD src)
 6380 %{
 6381   single_instruction;
 6382   src    : S1(read);
 6383   dst    : S5(write);
 6384   INS01  : ISS;
 6385   NEON_FP : S5;
 6386 %}
 6387 
 6388 pipe_class fp_f2d(vRegD dst, vRegF src)
 6389 %{
 6390   single_instruction;
 6391   src    : S1(read);
 6392   dst    : S5(write);
 6393   INS01  : ISS;
 6394   NEON_FP : S5;
 6395 %}
 6396 
 6397 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6398 %{
 6399   single_instruction;
 6400   src    : S1(read);
 6401   dst    : S5(write);
 6402   INS01  : ISS;
 6403   NEON_FP : S5;
 6404 %}
 6405 
 6406 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6407 %{
 6408   single_instruction;
 6409   src    : S1(read);
 6410   dst    : S5(write);
 6411   INS01  : ISS;
 6412   NEON_FP : S5;
 6413 %}
 6414 
 6415 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6416 %{
 6417   single_instruction;
 6418   src    : S1(read);
 6419   dst    : S5(write);
 6420   INS01  : ISS;
 6421   NEON_FP : S5;
 6422 %}
 6423 
 6424 pipe_class fp_l2f(vRegF dst, iRegL src)
 6425 %{
 6426   single_instruction;
 6427   src    : S1(read);
 6428   dst    : S5(write);
 6429   INS01  : ISS;
 6430   NEON_FP : S5;
 6431 %}
 6432 
 6433 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6434 %{
 6435   single_instruction;
 6436   src    : S1(read);
 6437   dst    : S5(write);
 6438   INS01  : ISS;
 6439   NEON_FP : S5;
 6440 %}
 6441 
 6442 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6443 %{
 6444   single_instruction;
 6445   src    : S1(read);
 6446   dst    : S5(write);
 6447   INS01  : ISS;
 6448   NEON_FP : S5;
 6449 %}
 6450 
 6451 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6452 %{
 6453   single_instruction;
 6454   src    : S1(read);
 6455   dst    : S5(write);
 6456   INS01  : ISS;
 6457   NEON_FP : S5;
 6458 %}
 6459 
 6460 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6461 %{
 6462   single_instruction;
 6463   src    : S1(read);
 6464   dst    : S5(write);
 6465   INS01  : ISS;
 6466   NEON_FP : S5;
 6467 %}
 6468 
 6469 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6470 %{
 6471   single_instruction;
 6472   src1   : S1(read);
 6473   src2   : S2(read);
 6474   dst    : S5(write);
 6475   INS0   : ISS;
 6476   NEON_FP : S5;
 6477 %}
 6478 
 6479 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6480 %{
 6481   single_instruction;
 6482   src1   : S1(read);
 6483   src2   : S2(read);
 6484   dst    : S5(write);
 6485   INS0   : ISS;
 6486   NEON_FP : S5;
 6487 %}
 6488 
 6489 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6490 %{
 6491   single_instruction;
 6492   cr     : S1(read);
 6493   src1   : S1(read);
 6494   src2   : S1(read);
 6495   dst    : S3(write);
 6496   INS01  : ISS;
 6497   NEON_FP : S3;
 6498 %}
 6499 
 6500 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6501 %{
 6502   single_instruction;
 6503   cr     : S1(read);
 6504   src1   : S1(read);
 6505   src2   : S1(read);
 6506   dst    : S3(write);
 6507   INS01  : ISS;
 6508   NEON_FP : S3;
 6509 %}
 6510 
 6511 pipe_class fp_imm_s(vRegF dst)
 6512 %{
 6513   single_instruction;
 6514   dst    : S3(write);
 6515   INS01  : ISS;
 6516   NEON_FP : S3;
 6517 %}
 6518 
 6519 pipe_class fp_imm_d(vRegD dst)
 6520 %{
 6521   single_instruction;
 6522   dst    : S3(write);
 6523   INS01  : ISS;
 6524   NEON_FP : S3;
 6525 %}
 6526 
 6527 pipe_class fp_load_constant_s(vRegF dst)
 6528 %{
 6529   single_instruction;
 6530   dst    : S4(write);
 6531   INS01  : ISS;
 6532   NEON_FP : S4;
 6533 %}
 6534 
 6535 pipe_class fp_load_constant_d(vRegD dst)
 6536 %{
 6537   single_instruction;
 6538   dst    : S4(write);
 6539   INS01  : ISS;
 6540   NEON_FP : S4;
 6541 %}
 6542 
 6543 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6544 %{
 6545   single_instruction;
 6546   dst    : S5(write);
 6547   src1   : S1(read);
 6548   src2   : S1(read);
 6549   INS01  : ISS;
 6550   NEON_FP : S5;
 6551 %}
 6552 
 6553 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6554 %{
 6555   single_instruction;
 6556   dst    : S5(write);
 6557   src1   : S1(read);
 6558   src2   : S1(read);
 6559   INS0   : ISS;
 6560   NEON_FP : S5;
 6561 %}
 6562 
 6563 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6564 %{
 6565   single_instruction;
 6566   dst    : S5(write);
 6567   src1   : S1(read);
 6568   src2   : S1(read);
 6569   dst    : S1(read);
 6570   INS01  : ISS;
 6571   NEON_FP : S5;
 6572 %}
 6573 
 6574 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6575 %{
 6576   single_instruction;
 6577   dst    : S5(write);
 6578   src1   : S1(read);
 6579   src2   : S1(read);
 6580   dst    : S1(read);
 6581   INS0   : ISS;
 6582   NEON_FP : S5;
 6583 %}
 6584 
 6585 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6586 %{
 6587   single_instruction;
 6588   dst    : S4(write);
 6589   src1   : S2(read);
 6590   src2   : S2(read);
 6591   INS01  : ISS;
 6592   NEON_FP : S4;
 6593 %}
 6594 
 6595 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6596 %{
 6597   single_instruction;
 6598   dst    : S4(write);
 6599   src1   : S2(read);
 6600   src2   : S2(read);
 6601   INS0   : ISS;
 6602   NEON_FP : S4;
 6603 %}
 6604 
 6605 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6606 %{
 6607   single_instruction;
 6608   dst    : S3(write);
 6609   src1   : S2(read);
 6610   src2   : S2(read);
 6611   INS01  : ISS;
 6612   NEON_FP : S3;
 6613 %}
 6614 
 6615 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6616 %{
 6617   single_instruction;
 6618   dst    : S3(write);
 6619   src1   : S2(read);
 6620   src2   : S2(read);
 6621   INS0   : ISS;
 6622   NEON_FP : S3;
 6623 %}
 6624 
 6625 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6626 %{
 6627   single_instruction;
 6628   dst    : S3(write);
 6629   src    : S1(read);
 6630   shift  : S1(read);
 6631   INS01  : ISS;
 6632   NEON_FP : S3;
 6633 %}
 6634 
 6635 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6636 %{
 6637   single_instruction;
 6638   dst    : S3(write);
 6639   src    : S1(read);
 6640   shift  : S1(read);
 6641   INS0   : ISS;
 6642   NEON_FP : S3;
 6643 %}
 6644 
 6645 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6646 %{
 6647   single_instruction;
 6648   dst    : S3(write);
 6649   src    : S1(read);
 6650   INS01  : ISS;
 6651   NEON_FP : S3;
 6652 %}
 6653 
 6654 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6655 %{
 6656   single_instruction;
 6657   dst    : S3(write);
 6658   src    : S1(read);
 6659   INS0   : ISS;
 6660   NEON_FP : S3;
 6661 %}
 6662 
 6663 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6664 %{
 6665   single_instruction;
 6666   dst    : S5(write);
 6667   src1   : S1(read);
 6668   src2   : S1(read);
 6669   INS01  : ISS;
 6670   NEON_FP : S5;
 6671 %}
 6672 
 6673 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6674 %{
 6675   single_instruction;
 6676   dst    : S5(write);
 6677   src1   : S1(read);
 6678   src2   : S1(read);
 6679   INS0   : ISS;
 6680   NEON_FP : S5;
 6681 %}
 6682 
 6683 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6684 %{
 6685   single_instruction;
 6686   dst    : S5(write);
 6687   src1   : S1(read);
 6688   src2   : S1(read);
 6689   INS0   : ISS;
 6690   NEON_FP : S5;
 6691 %}
 6692 
 6693 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6694 %{
 6695   single_instruction;
 6696   dst    : S5(write);
 6697   src1   : S1(read);
 6698   src2   : S1(read);
 6699   INS0   : ISS;
 6700   NEON_FP : S5;
 6701 %}
 6702 
 6703 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6704 %{
 6705   single_instruction;
 6706   dst    : S5(write);
 6707   src    : S1(read);
 6708   INS0   : ISS;
 6709   NEON_FP : S5;
 6710 %}
 6711 
 6712 pipe_class vunop_fp64(vecD dst, vecD src)
 6713 %{
 6714   single_instruction;
 6715   dst    : S5(write);
 6716   src    : S1(read);
 6717   INS01  : ISS;
 6718   NEON_FP : S5;
 6719 %}
 6720 
 6721 pipe_class vunop_fp128(vecX dst, vecX src)
 6722 %{
 6723   single_instruction;
 6724   dst    : S5(write);
 6725   src    : S1(read);
 6726   INS0   : ISS;
 6727   NEON_FP : S5;
 6728 %}
 6729 
 6730 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6731 %{
 6732   single_instruction;
 6733   dst    : S3(write);
 6734   src    : S1(read);
 6735   INS01  : ISS;
 6736   NEON_FP : S3;
 6737 %}
 6738 
 6739 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6740 %{
 6741   single_instruction;
 6742   dst    : S3(write);
 6743   src    : S1(read);
 6744   INS01  : ISS;
 6745   NEON_FP : S3;
 6746 %}
 6747 
 6748 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6749 %{
 6750   single_instruction;
 6751   dst    : S3(write);
 6752   src    : S1(read);
 6753   INS01  : ISS;
 6754   NEON_FP : S3;
 6755 %}
 6756 
 6757 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6758 %{
 6759   single_instruction;
 6760   dst    : S3(write);
 6761   src    : S1(read);
 6762   INS01  : ISS;
 6763   NEON_FP : S3;
 6764 %}
 6765 
 6766 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6767 %{
 6768   single_instruction;
 6769   dst    : S3(write);
 6770   src    : S1(read);
 6771   INS01  : ISS;
 6772   NEON_FP : S3;
 6773 %}
 6774 
 6775 pipe_class vmovi_reg_imm64(vecD dst)
 6776 %{
 6777   single_instruction;
 6778   dst    : S3(write);
 6779   INS01  : ISS;
 6780   NEON_FP : S3;
 6781 %}
 6782 
 6783 pipe_class vmovi_reg_imm128(vecX dst)
 6784 %{
 6785   single_instruction;
 6786   dst    : S3(write);
 6787   INS0   : ISS;
 6788   NEON_FP : S3;
 6789 %}
 6790 
 6791 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6792 %{
 6793   single_instruction;
 6794   dst    : S5(write);
 6795   mem    : ISS(read);
 6796   INS01  : ISS;
 6797   NEON_FP : S3;
 6798 %}
 6799 
 6800 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6801 %{
 6802   single_instruction;
 6803   dst    : S5(write);
 6804   mem    : ISS(read);
 6805   INS01  : ISS;
 6806   NEON_FP : S3;
 6807 %}
 6808 
 6809 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6810 %{
 6811   single_instruction;
 6812   mem    : ISS(read);
 6813   src    : S2(read);
 6814   INS01  : ISS;
 6815   NEON_FP : S3;
 6816 %}
 6817 
 6818 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6819 %{
 6820   single_instruction;
 6821   mem    : ISS(read);
 6822   src    : S2(read);
 6823   INS01  : ISS;
 6824   NEON_FP : S3;
 6825 %}
 6826 
 6827 //------- Integer ALU operations --------------------------
 6828 
 6829 // Integer ALU reg-reg operation
 6830 // Operands needed in EX1, result generated in EX2
 6831 // Eg.  ADD     x0, x1, x2
 6832 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6833 %{
 6834   single_instruction;
 6835   dst    : EX2(write);
 6836   src1   : EX1(read);
 6837   src2   : EX1(read);
 6838   INS01  : ISS; // Dual issue as instruction 0 or 1
 6839   ALU    : EX2;
 6840 %}
 6841 
 6842 // Integer ALU reg-reg operation with constant shift
 6843 // Shifted register must be available in LATE_ISS instead of EX1
 6844 // Eg.  ADD     x0, x1, x2, LSL #2
 6845 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6846 %{
 6847   single_instruction;
 6848   dst    : EX2(write);
 6849   src1   : EX1(read);
 6850   src2   : ISS(read);
 6851   INS01  : ISS;
 6852   ALU    : EX2;
 6853 %}
 6854 
 6855 // Integer ALU reg operation with constant shift
 6856 // Eg.  LSL     x0, x1, #shift
 6857 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6858 %{
 6859   single_instruction;
 6860   dst    : EX2(write);
 6861   src1   : ISS(read);
 6862   INS01  : ISS;
 6863   ALU    : EX2;
 6864 %}
 6865 
 6866 // Integer ALU reg-reg operation with variable shift
 6867 // Both operands must be available in LATE_ISS instead of EX1
 6868 // Result is available in EX1 instead of EX2
 6869 // Eg.  LSLV    x0, x1, x2
 6870 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6871 %{
 6872   single_instruction;
 6873   dst    : EX1(write);
 6874   src1   : ISS(read);
 6875   src2   : ISS(read);
 6876   INS01  : ISS;
 6877   ALU    : EX1;
 6878 %}
 6879 
 6880 // Integer ALU reg-reg operation with extract
 6881 // As for _vshift above, but result generated in EX2
 6882 // Eg.  EXTR    x0, x1, x2, #N
 6883 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6884 %{
 6885   single_instruction;
 6886   dst    : EX2(write);
 6887   src1   : ISS(read);
 6888   src2   : ISS(read);
 6889   INS1   : ISS; // Can only dual issue as Instruction 1
 6890   ALU    : EX1;
 6891 %}
 6892 
 6893 // Integer ALU reg operation
 6894 // Eg.  NEG     x0, x1
 6895 pipe_class ialu_reg(iRegI dst, iRegI src)
 6896 %{
 6897   single_instruction;
 6898   dst    : EX2(write);
 6899   src    : EX1(read);
 6900   INS01  : ISS;
 6901   ALU    : EX2;
 6902 %}
 6903 
 6904 // Integer ALU reg mmediate operation
 6905 // Eg.  ADD     x0, x1, #N
 6906 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6907 %{
 6908   single_instruction;
 6909   dst    : EX2(write);
 6910   src1   : EX1(read);
 6911   INS01  : ISS;
 6912   ALU    : EX2;
 6913 %}
 6914 
 6915 // Integer ALU immediate operation (no source operands)
 6916 // Eg.  MOV     x0, #N
 6917 pipe_class ialu_imm(iRegI dst)
 6918 %{
 6919   single_instruction;
 6920   dst    : EX1(write);
 6921   INS01  : ISS;
 6922   ALU    : EX1;
 6923 %}
 6924 
 6925 //------- Compare operation -------------------------------
 6926 
 6927 // Compare reg-reg
 6928 // Eg.  CMP     x0, x1
 6929 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6930 %{
 6931   single_instruction;
 6932 //  fixed_latency(16);
 6933   cr     : EX2(write);
 6934   op1    : EX1(read);
 6935   op2    : EX1(read);
 6936   INS01  : ISS;
 6937   ALU    : EX2;
 6938 %}
 6939 
 6940 // Compare reg-reg
 6941 // Eg.  CMP     x0, #N
 6942 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6943 %{
 6944   single_instruction;
 6945 //  fixed_latency(16);
 6946   cr     : EX2(write);
 6947   op1    : EX1(read);
 6948   INS01  : ISS;
 6949   ALU    : EX2;
 6950 %}
 6951 
 6952 //------- Conditional instructions ------------------------
 6953 
 6954 // Conditional no operands
 6955 // Eg.  CSINC   x0, zr, zr, <cond>
 6956 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6957 %{
 6958   single_instruction;
 6959   cr     : EX1(read);
 6960   dst    : EX2(write);
 6961   INS01  : ISS;
 6962   ALU    : EX2;
 6963 %}
 6964 
 6965 // Conditional 2 operand
 6966 // EG.  CSEL    X0, X1, X2, <cond>
 6967 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6968 %{
 6969   single_instruction;
 6970   cr     : EX1(read);
 6971   src1   : EX1(read);
 6972   src2   : EX1(read);
 6973   dst    : EX2(write);
 6974   INS01  : ISS;
 6975   ALU    : EX2;
 6976 %}
 6977 
 6978 // Conditional 2 operand
 6979 // EG.  CSEL    X0, X1, X2, <cond>
 6980 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6981 %{
 6982   single_instruction;
 6983   cr     : EX1(read);
 6984   src    : EX1(read);
 6985   dst    : EX2(write);
 6986   INS01  : ISS;
 6987   ALU    : EX2;
 6988 %}
 6989 
 6990 //------- Multiply pipeline operations --------------------
 6991 
 6992 // Multiply reg-reg
 6993 // Eg.  MUL     w0, w1, w2
 6994 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6995 %{
 6996   single_instruction;
 6997   dst    : WR(write);
 6998   src1   : ISS(read);
 6999   src2   : ISS(read);
 7000   INS01  : ISS;
 7001   MAC    : WR;
 7002 %}
 7003 
 7004 // Multiply accumulate
 7005 // Eg.  MADD    w0, w1, w2, w3
 7006 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7007 %{
 7008   single_instruction;
 7009   dst    : WR(write);
 7010   src1   : ISS(read);
 7011   src2   : ISS(read);
 7012   src3   : ISS(read);
 7013   INS01  : ISS;
 7014   MAC    : WR;
 7015 %}
 7016 
 7017 // Eg.  MUL     w0, w1, w2
 7018 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7019 %{
 7020   single_instruction;
 7021   fixed_latency(3); // Maximum latency for 64 bit mul
 7022   dst    : WR(write);
 7023   src1   : ISS(read);
 7024   src2   : ISS(read);
 7025   INS01  : ISS;
 7026   MAC    : WR;
 7027 %}
 7028 
 7029 // Multiply accumulate
 7030 // Eg.  MADD    w0, w1, w2, w3
 7031 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7032 %{
 7033   single_instruction;
 7034   fixed_latency(3); // Maximum latency for 64 bit mul
 7035   dst    : WR(write);
 7036   src1   : ISS(read);
 7037   src2   : ISS(read);
 7038   src3   : ISS(read);
 7039   INS01  : ISS;
 7040   MAC    : WR;
 7041 %}
 7042 
 7043 //------- Divide pipeline operations --------------------
 7044 
 7045 // Eg.  SDIV    w0, w1, w2
 7046 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7047 %{
 7048   single_instruction;
 7049   fixed_latency(8); // Maximum latency for 32 bit divide
 7050   dst    : WR(write);
 7051   src1   : ISS(read);
 7052   src2   : ISS(read);
 7053   INS0   : ISS; // Can only dual issue as instruction 0
 7054   DIV    : WR;
 7055 %}
 7056 
 7057 // Eg.  SDIV    x0, x1, x2
 7058 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7059 %{
 7060   single_instruction;
 7061   fixed_latency(16); // Maximum latency for 64 bit divide
 7062   dst    : WR(write);
 7063   src1   : ISS(read);
 7064   src2   : ISS(read);
 7065   INS0   : ISS; // Can only dual issue as instruction 0
 7066   DIV    : WR;
 7067 %}
 7068 
 7069 //------- Load pipeline operations ------------------------
 7070 
 7071 // Load - prefetch
 7072 // Eg.  PFRM    <mem>
 7073 pipe_class iload_prefetch(memory mem)
 7074 %{
 7075   single_instruction;
 7076   mem    : ISS(read);
 7077   INS01  : ISS;
 7078   LDST   : WR;
 7079 %}
 7080 
 7081 // Load - reg, mem
 7082 // Eg.  LDR     x0, <mem>
 7083 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7084 %{
 7085   single_instruction;
 7086   dst    : WR(write);
 7087   mem    : ISS(read);
 7088   INS01  : ISS;
 7089   LDST   : WR;
 7090 %}
 7091 
 7092 // Load - reg, reg
 7093 // Eg.  LDR     x0, [sp, x1]
 7094 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7095 %{
 7096   single_instruction;
 7097   dst    : WR(write);
 7098   src    : ISS(read);
 7099   INS01  : ISS;
 7100   LDST   : WR;
 7101 %}
 7102 
 7103 //------- Store pipeline operations -----------------------
 7104 
 7105 // Store - zr, mem
 7106 // Eg.  STR     zr, <mem>
 7107 pipe_class istore_mem(memory mem)
 7108 %{
 7109   single_instruction;
 7110   mem    : ISS(read);
 7111   INS01  : ISS;
 7112   LDST   : WR;
 7113 %}
 7114 
 7115 // Store - reg, mem
 7116 // Eg.  STR     x0, <mem>
 7117 pipe_class istore_reg_mem(iRegI src, memory mem)
 7118 %{
 7119   single_instruction;
 7120   mem    : ISS(read);
 7121   src    : EX2(read);
 7122   INS01  : ISS;
 7123   LDST   : WR;
 7124 %}
 7125 
 7126 // Store - reg, reg
 7127 // Eg. STR      x0, [sp, x1]
 7128 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7129 %{
 7130   single_instruction;
 7131   dst    : ISS(read);
 7132   src    : EX2(read);
 7133   INS01  : ISS;
 7134   LDST   : WR;
 7135 %}
 7136 
 7137 //------- Store pipeline operations -----------------------
 7138 
 7139 // Branch
 7140 pipe_class pipe_branch()
 7141 %{
 7142   single_instruction;
 7143   INS01  : ISS;
 7144   BRANCH : EX1;
 7145 %}
 7146 
 7147 // Conditional branch
 7148 pipe_class pipe_branch_cond(rFlagsReg cr)
 7149 %{
 7150   single_instruction;
 7151   cr     : EX1(read);
 7152   INS01  : ISS;
 7153   BRANCH : EX1;
 7154 %}
 7155 
 7156 // Compare & Branch
 7157 // EG.  CBZ/CBNZ
 7158 pipe_class pipe_cmp_branch(iRegI op1)
 7159 %{
 7160   single_instruction;
 7161   op1    : EX1(read);
 7162   INS01  : ISS;
 7163   BRANCH : EX1;
 7164 %}
 7165 
 7166 //------- Synchronisation operations ----------------------
 7167 
 7168 // Any operation requiring serialization.
 7169 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7170 pipe_class pipe_serial()
 7171 %{
 7172   single_instruction;
 7173   force_serialization;
 7174   fixed_latency(16);
 7175   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7176   LDST   : WR;
 7177 %}
 7178 
 7179 // Generic big/slow expanded idiom - also serialized
 7180 pipe_class pipe_slow()
 7181 %{
 7182   instruction_count(10);
 7183   multiple_bundles;
 7184   force_serialization;
 7185   fixed_latency(16);
 7186   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7187   LDST   : WR;
 7188 %}
 7189 
 7190 // Empty pipeline class
 7191 pipe_class pipe_class_empty()
 7192 %{
 7193   single_instruction;
 7194   fixed_latency(0);
 7195 %}
 7196 
 7197 // Default pipeline class.
 7198 pipe_class pipe_class_default()
 7199 %{
 7200   single_instruction;
 7201   fixed_latency(2);
 7202 %}
 7203 
 7204 // Pipeline class for compares.
 7205 pipe_class pipe_class_compare()
 7206 %{
 7207   single_instruction;
 7208   fixed_latency(16);
 7209 %}
 7210 
 7211 // Pipeline class for memory operations.
 7212 pipe_class pipe_class_memory()
 7213 %{
 7214   single_instruction;
 7215   fixed_latency(16);
 7216 %}
 7217 
 7218 // Pipeline class for call.
 7219 pipe_class pipe_class_call()
 7220 %{
 7221   single_instruction;
 7222   fixed_latency(100);
 7223 %}
 7224 
 7225 // Define the class for the Nop node.
 7226 define %{
 7227    MachNop = pipe_class_empty;
 7228 %}
 7229 
 7230 %}
 7231 //----------INSTRUCTIONS-------------------------------------------------------
 7232 //
 7233 // match      -- States which machine-independent subtree may be replaced
 7234 //               by this instruction.
 7235 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7236 //               selection to identify a minimum cost tree of machine
 7237 //               instructions that matches a tree of machine-independent
 7238 //               instructions.
 7239 // format     -- A string providing the disassembly for this instruction.
 7240 //               The value of an instruction's operand may be inserted
 7241 //               by referring to it with a '$' prefix.
 7242 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7243 //               to within an encode class as $primary, $secondary, and $tertiary
 7244 //               rrspectively.  The primary opcode is commonly used to
 7245 //               indicate the type of machine instruction, while secondary
 7246 //               and tertiary are often used for prefix options or addressing
 7247 //               modes.
 7248 // ins_encode -- A list of encode classes with parameters. The encode class
 7249 //               name must have been defined in an 'enc_class' specification
 7250 //               in the encode section of the architecture description.
 7251 
 7252 // ============================================================================
 7253 // Memory (Load/Store) Instructions
 7254 
 7255 // Load Instructions
 7256 
 7257 // Load Byte (8 bit signed)
 7258 instruct loadB(iRegINoSp dst, memory1 mem)
 7259 %{
 7260   match(Set dst (LoadB mem));
 7261   predicate(!needs_acquiring_load(n));
 7262 
 7263   ins_cost(4 * INSN_COST);
 7264   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7265 
 7266   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7267 
 7268   ins_pipe(iload_reg_mem);
 7269 %}
 7270 
 7271 // Load Byte (8 bit signed) into long
 7272 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7273 %{
 7274   match(Set dst (ConvI2L (LoadB mem)));
 7275   predicate(!needs_acquiring_load(n->in(1)));
 7276 
 7277   ins_cost(4 * INSN_COST);
 7278   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7279 
 7280   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7281 
 7282   ins_pipe(iload_reg_mem);
 7283 %}
 7284 
 7285 // Load Byte (8 bit unsigned)
 7286 instruct loadUB(iRegINoSp dst, memory1 mem)
 7287 %{
 7288   match(Set dst (LoadUB mem));
 7289   predicate(!needs_acquiring_load(n));
 7290 
 7291   ins_cost(4 * INSN_COST);
 7292   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7293 
 7294   ins_encode(aarch64_enc_ldrb(dst, mem));
 7295 
 7296   ins_pipe(iload_reg_mem);
 7297 %}
 7298 
 7299 // Load Byte (8 bit unsigned) into long
 7300 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7301 %{
 7302   match(Set dst (ConvI2L (LoadUB mem)));
 7303   predicate(!needs_acquiring_load(n->in(1)));
 7304 
 7305   ins_cost(4 * INSN_COST);
 7306   format %{ "ldrb  $dst, $mem\t# byte" %}
 7307 
 7308   ins_encode(aarch64_enc_ldrb(dst, mem));
 7309 
 7310   ins_pipe(iload_reg_mem);
 7311 %}
 7312 
 7313 // Load Short (16 bit signed)
 7314 instruct loadS(iRegINoSp dst, memory2 mem)
 7315 %{
 7316   match(Set dst (LoadS mem));
 7317   predicate(!needs_acquiring_load(n));
 7318 
 7319   ins_cost(4 * INSN_COST);
 7320   format %{ "ldrshw  $dst, $mem\t# short" %}
 7321 
 7322   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7323 
 7324   ins_pipe(iload_reg_mem);
 7325 %}
 7326 
 7327 // Load Short (16 bit signed) into long
 7328 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7329 %{
 7330   match(Set dst (ConvI2L (LoadS mem)));
 7331   predicate(!needs_acquiring_load(n->in(1)));
 7332 
 7333   ins_cost(4 * INSN_COST);
 7334   format %{ "ldrsh  $dst, $mem\t# short" %}
 7335 
 7336   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7337 
 7338   ins_pipe(iload_reg_mem);
 7339 %}
 7340 
 7341 // Load Char (16 bit unsigned)
 7342 instruct loadUS(iRegINoSp dst, memory2 mem)
 7343 %{
 7344   match(Set dst (LoadUS mem));
 7345   predicate(!needs_acquiring_load(n));
 7346 
 7347   ins_cost(4 * INSN_COST);
 7348   format %{ "ldrh  $dst, $mem\t# short" %}
 7349 
 7350   ins_encode(aarch64_enc_ldrh(dst, mem));
 7351 
 7352   ins_pipe(iload_reg_mem);
 7353 %}
 7354 
 7355 // Load Short/Char (16 bit unsigned) into long
 7356 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7357 %{
 7358   match(Set dst (ConvI2L (LoadUS mem)));
 7359   predicate(!needs_acquiring_load(n->in(1)));
 7360 
 7361   ins_cost(4 * INSN_COST);
 7362   format %{ "ldrh  $dst, $mem\t# short" %}
 7363 
 7364   ins_encode(aarch64_enc_ldrh(dst, mem));
 7365 
 7366   ins_pipe(iload_reg_mem);
 7367 %}
 7368 
 7369 // Load Integer (32 bit signed)
 7370 instruct loadI(iRegINoSp dst, memory4 mem)
 7371 %{
 7372   match(Set dst (LoadI mem));
 7373   predicate(!needs_acquiring_load(n));
 7374 
 7375   ins_cost(4 * INSN_COST);
 7376   format %{ "ldrw  $dst, $mem\t# int" %}
 7377 
 7378   ins_encode(aarch64_enc_ldrw(dst, mem));
 7379 
 7380   ins_pipe(iload_reg_mem);
 7381 %}
 7382 
 7383 // Load Integer (32 bit signed) into long
 7384 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7385 %{
 7386   match(Set dst (ConvI2L (LoadI mem)));
 7387   predicate(!needs_acquiring_load(n->in(1)));
 7388 
 7389   ins_cost(4 * INSN_COST);
 7390   format %{ "ldrsw  $dst, $mem\t# int" %}
 7391 
 7392   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7393 
 7394   ins_pipe(iload_reg_mem);
 7395 %}
 7396 
 7397 // Load Integer (32 bit unsigned) into long
 7398 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7399 %{
 7400   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7401   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7402 
 7403   ins_cost(4 * INSN_COST);
 7404   format %{ "ldrw  $dst, $mem\t# int" %}
 7405 
 7406   ins_encode(aarch64_enc_ldrw(dst, mem));
 7407 
 7408   ins_pipe(iload_reg_mem);
 7409 %}
 7410 
 7411 // Load Long (64 bit signed)
 7412 instruct loadL(iRegLNoSp dst, memory8 mem)
 7413 %{
 7414   match(Set dst (LoadL mem));
 7415   predicate(!needs_acquiring_load(n));
 7416 
 7417   ins_cost(4 * INSN_COST);
 7418   format %{ "ldr  $dst, $mem\t# int" %}
 7419 
 7420   ins_encode(aarch64_enc_ldr(dst, mem));
 7421 
 7422   ins_pipe(iload_reg_mem);
 7423 %}
 7424 
 7425 // Load Range
 7426 instruct loadRange(iRegINoSp dst, memory4 mem)
 7427 %{
 7428   match(Set dst (LoadRange mem));
 7429 
 7430   ins_cost(4 * INSN_COST);
 7431   format %{ "ldrw  $dst, $mem\t# range" %}
 7432 
 7433   ins_encode(aarch64_enc_ldrw(dst, mem));
 7434 
 7435   ins_pipe(iload_reg_mem);
 7436 %}
 7437 
 7438 // Load Pointer
 7439 instruct loadP(iRegPNoSp dst, memory8 mem)
 7440 %{
 7441   match(Set dst (LoadP mem));
 7442   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7443 
 7444   ins_cost(4 * INSN_COST);
 7445   format %{ "ldr  $dst, $mem\t# ptr" %}
 7446 
 7447   ins_encode(aarch64_enc_ldr(dst, mem));
 7448 
 7449   ins_pipe(iload_reg_mem);
 7450 %}
 7451 
 7452 // Load Compressed Pointer
 7453 instruct loadN(iRegNNoSp dst, memory4 mem)
 7454 %{
 7455   match(Set dst (LoadN mem));
 7456   predicate(!needs_acquiring_load(n));
 7457 
 7458   ins_cost(4 * INSN_COST);
 7459   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7460 
 7461   ins_encode(aarch64_enc_ldrw(dst, mem));
 7462 
 7463   ins_pipe(iload_reg_mem);
 7464 %}
 7465 
 7466 // Load Klass Pointer
 7467 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7468 %{
 7469   match(Set dst (LoadKlass mem));
 7470   predicate(!needs_acquiring_load(n));
 7471 
 7472   ins_cost(4 * INSN_COST);
 7473   format %{ "ldr  $dst, $mem\t# class" %}
 7474 
 7475   ins_encode(aarch64_enc_ldr(dst, mem));
 7476 
 7477   ins_pipe(iload_reg_mem);
 7478 %}
 7479 
 7480 // Load Narrow Klass Pointer
 7481 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7482 %{
 7483   match(Set dst (LoadNKlass mem));
 7484   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 7485 
 7486   ins_cost(4 * INSN_COST);
 7487   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7488 
 7489   ins_encode(aarch64_enc_ldrw(dst, mem));
 7490 
 7491   ins_pipe(iload_reg_mem);
 7492 %}
 7493 
 7494 instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7495 %{
 7496   match(Set dst (LoadNKlass mem));
 7497   effect(TEMP_DEF dst, KILL cr);
 7498   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 7499 
 7500   ins_cost(4 * INSN_COST);
 7501   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7502   ins_encode %{
 7503     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7504     assert($mem$$index$$Register == noreg, "expect no index");
 7505     Register dst = $dst$$Register;
 7506     Register obj = $mem$$base$$Register;
 7507     C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
 7508     Compile::current()->output()->add_stub(stub);
 7509     __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
 7510     // NOTE: We can't use tbnz here, because the target is sometimes too far away
 7511     // and cannot be encoded.
 7512     __ tst(dst, markWord::monitor_value);
 7513     __ br(Assembler::NE, stub->entry());
 7514     __ bind(stub->continuation());
 7515     __ lsr(dst, dst, markWord::klass_shift);
 7516   %}
 7517   ins_pipe(pipe_slow);
 7518 %}
 7519 
 7520 // Load Float
 7521 instruct loadF(vRegF dst, memory4 mem)
 7522 %{
 7523   match(Set dst (LoadF mem));
 7524   predicate(!needs_acquiring_load(n));
 7525 
 7526   ins_cost(4 * INSN_COST);
 7527   format %{ "ldrs  $dst, $mem\t# float" %}
 7528 
 7529   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7530 
 7531   ins_pipe(pipe_class_memory);
 7532 %}
 7533 
 7534 // Load Double
 7535 instruct loadD(vRegD dst, memory8 mem)
 7536 %{
 7537   match(Set dst (LoadD mem));
 7538   predicate(!needs_acquiring_load(n));
 7539 
 7540   ins_cost(4 * INSN_COST);
 7541   format %{ "ldrd  $dst, $mem\t# double" %}
 7542 
 7543   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7544 
 7545   ins_pipe(pipe_class_memory);
 7546 %}
 7547 
 7548 
 7549 // Load Int Constant
 7550 instruct loadConI(iRegINoSp dst, immI src)
 7551 %{
 7552   match(Set dst src);
 7553 
 7554   ins_cost(INSN_COST);
 7555   format %{ "mov $dst, $src\t# int" %}
 7556 
 7557   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7558 
 7559   ins_pipe(ialu_imm);
 7560 %}
 7561 
 7562 // Load Long Constant
 7563 instruct loadConL(iRegLNoSp dst, immL src)
 7564 %{
 7565   match(Set dst src);
 7566 
 7567   ins_cost(INSN_COST);
 7568   format %{ "mov $dst, $src\t# long" %}
 7569 
 7570   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7571 
 7572   ins_pipe(ialu_imm);
 7573 %}
 7574 
 7575 // Load Pointer Constant
 7576 
 7577 instruct loadConP(iRegPNoSp dst, immP con)
 7578 %{
 7579   match(Set dst con);
 7580 
 7581   ins_cost(INSN_COST * 4);
 7582   format %{
 7583     "mov  $dst, $con\t# ptr\n\t"
 7584   %}
 7585 
 7586   ins_encode(aarch64_enc_mov_p(dst, con));
 7587 
 7588   ins_pipe(ialu_imm);
 7589 %}
 7590 
 7591 // Load Null Pointer Constant
 7592 
 7593 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7594 %{
 7595   match(Set dst con);
 7596 
 7597   ins_cost(INSN_COST);
 7598   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7599 
 7600   ins_encode(aarch64_enc_mov_p0(dst, con));
 7601 
 7602   ins_pipe(ialu_imm);
 7603 %}
 7604 
 7605 // Load Pointer Constant One
 7606 
 7607 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7608 %{
 7609   match(Set dst con);
 7610 
 7611   ins_cost(INSN_COST);
 7612   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7613 
 7614   ins_encode(aarch64_enc_mov_p1(dst, con));
 7615 
 7616   ins_pipe(ialu_imm);
 7617 %}
 7618 
 7619 // Load Byte Map Base Constant
 7620 
 7621 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7622 %{
 7623   match(Set dst con);
 7624 
 7625   ins_cost(INSN_COST);
 7626   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7627 
 7628   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7629 
 7630   ins_pipe(ialu_imm);
 7631 %}
 7632 
 7633 // Load Narrow Pointer Constant
 7634 
 7635 instruct loadConN(iRegNNoSp dst, immN con)
 7636 %{
 7637   match(Set dst con);
 7638 
 7639   ins_cost(INSN_COST * 4);
 7640   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7641 
 7642   ins_encode(aarch64_enc_mov_n(dst, con));
 7643 
 7644   ins_pipe(ialu_imm);
 7645 %}
 7646 
 7647 // Load Narrow Null Pointer Constant
 7648 
 7649 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7650 %{
 7651   match(Set dst con);
 7652 
 7653   ins_cost(INSN_COST);
 7654   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7655 
 7656   ins_encode(aarch64_enc_mov_n0(dst, con));
 7657 
 7658   ins_pipe(ialu_imm);
 7659 %}
 7660 
 7661 // Load Narrow Klass Constant
 7662 
 7663 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7664 %{
 7665   match(Set dst con);
 7666 
 7667   ins_cost(INSN_COST);
 7668   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7669 
 7670   ins_encode(aarch64_enc_mov_nk(dst, con));
 7671 
 7672   ins_pipe(ialu_imm);
 7673 %}
 7674 
 7675 // Load Packed Float Constant
 7676 
 7677 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7678   match(Set dst con);
 7679   ins_cost(INSN_COST * 4);
 7680   format %{ "fmovs  $dst, $con"%}
 7681   ins_encode %{
 7682     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7683   %}
 7684 
 7685   ins_pipe(fp_imm_s);
 7686 %}
 7687 
 7688 // Load Float Constant
 7689 
 7690 instruct loadConF(vRegF dst, immF con) %{
 7691   match(Set dst con);
 7692 
 7693   ins_cost(INSN_COST * 4);
 7694 
 7695   format %{
 7696     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7697   %}
 7698 
 7699   ins_encode %{
 7700     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7701   %}
 7702 
 7703   ins_pipe(fp_load_constant_s);
 7704 %}
 7705 
 7706 // Load Packed Double Constant
 7707 
 7708 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7709   match(Set dst con);
 7710   ins_cost(INSN_COST);
 7711   format %{ "fmovd  $dst, $con"%}
 7712   ins_encode %{
 7713     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7714   %}
 7715 
 7716   ins_pipe(fp_imm_d);
 7717 %}
 7718 
 7719 // Load Double Constant
 7720 
 7721 instruct loadConD(vRegD dst, immD con) %{
 7722   match(Set dst con);
 7723 
 7724   ins_cost(INSN_COST * 5);
 7725   format %{
 7726     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7727   %}
 7728 
 7729   ins_encode %{
 7730     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7731   %}
 7732 
 7733   ins_pipe(fp_load_constant_d);
 7734 %}
 7735 
 7736 // Store Instructions
 7737 
 7738 // Store CMS card-mark Immediate
 7739 instruct storeimmCM0(immI0 zero, memory1 mem)
 7740 %{
 7741   match(Set mem (StoreCM mem zero));
 7742 
 7743   ins_cost(INSN_COST);
 7744   format %{ "storestore (elided)\n\t"
 7745             "strb zr, $mem\t# byte" %}
 7746 
 7747   ins_encode(aarch64_enc_strb0(mem));
 7748 
 7749   ins_pipe(istore_mem);
 7750 %}
 7751 
 7752 // Store CMS card-mark Immediate with intervening StoreStore
 7753 // needed when using CMS with no conditional card marking
 7754 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7755 %{
 7756   match(Set mem (StoreCM mem zero));
 7757 
 7758   ins_cost(INSN_COST * 2);
 7759   format %{ "storestore\n\t"
 7760             "dmb ishst"
 7761             "\n\tstrb zr, $mem\t# byte" %}
 7762 
 7763   ins_encode(aarch64_enc_strb0_ordered(mem));
 7764 
 7765   ins_pipe(istore_mem);
 7766 %}
 7767 
 7768 // Store Byte
 7769 instruct storeB(iRegIorL2I src, memory1 mem)
 7770 %{
 7771   match(Set mem (StoreB mem src));
 7772   predicate(!needs_releasing_store(n));
 7773 
 7774   ins_cost(INSN_COST);
 7775   format %{ "strb  $src, $mem\t# byte" %}
 7776 
 7777   ins_encode(aarch64_enc_strb(src, mem));
 7778 
 7779   ins_pipe(istore_reg_mem);
 7780 %}
 7781 
 7782 
 7783 instruct storeimmB0(immI0 zero, memory1 mem)
 7784 %{
 7785   match(Set mem (StoreB mem zero));
 7786   predicate(!needs_releasing_store(n));
 7787 
 7788   ins_cost(INSN_COST);
 7789   format %{ "strb rscractch2, $mem\t# byte" %}
 7790 
 7791   ins_encode(aarch64_enc_strb0(mem));
 7792 
 7793   ins_pipe(istore_mem);
 7794 %}
 7795 
 7796 // Store Char/Short
 7797 instruct storeC(iRegIorL2I src, memory2 mem)
 7798 %{
 7799   match(Set mem (StoreC mem src));
 7800   predicate(!needs_releasing_store(n));
 7801 
 7802   ins_cost(INSN_COST);
 7803   format %{ "strh  $src, $mem\t# short" %}
 7804 
 7805   ins_encode(aarch64_enc_strh(src, mem));
 7806 
 7807   ins_pipe(istore_reg_mem);
 7808 %}
 7809 
 7810 instruct storeimmC0(immI0 zero, memory2 mem)
 7811 %{
 7812   match(Set mem (StoreC mem zero));
 7813   predicate(!needs_releasing_store(n));
 7814 
 7815   ins_cost(INSN_COST);
 7816   format %{ "strh  zr, $mem\t# short" %}
 7817 
 7818   ins_encode(aarch64_enc_strh0(mem));
 7819 
 7820   ins_pipe(istore_mem);
 7821 %}
 7822 
 7823 // Store Integer
 7824 
 7825 instruct storeI(iRegIorL2I src, memory4 mem)
 7826 %{
 7827   match(Set mem(StoreI mem src));
 7828   predicate(!needs_releasing_store(n));
 7829 
 7830   ins_cost(INSN_COST);
 7831   format %{ "strw  $src, $mem\t# int" %}
 7832 
 7833   ins_encode(aarch64_enc_strw(src, mem));
 7834 
 7835   ins_pipe(istore_reg_mem);
 7836 %}
 7837 
 7838 instruct storeimmI0(immI0 zero, memory4 mem)
 7839 %{
 7840   match(Set mem(StoreI mem zero));
 7841   predicate(!needs_releasing_store(n));
 7842 
 7843   ins_cost(INSN_COST);
 7844   format %{ "strw  zr, $mem\t# int" %}
 7845 
 7846   ins_encode(aarch64_enc_strw0(mem));
 7847 
 7848   ins_pipe(istore_mem);
 7849 %}
 7850 
 7851 // Store Long (64 bit signed)
 7852 instruct storeL(iRegL src, memory8 mem)
 7853 %{
 7854   match(Set mem (StoreL mem src));
 7855   predicate(!needs_releasing_store(n));
 7856 
 7857   ins_cost(INSN_COST);
 7858   format %{ "str  $src, $mem\t# int" %}
 7859 
 7860   ins_encode(aarch64_enc_str(src, mem));
 7861 
 7862   ins_pipe(istore_reg_mem);
 7863 %}
 7864 
 7865 // Store Long (64 bit signed)
 7866 instruct storeimmL0(immL0 zero, memory8 mem)
 7867 %{
 7868   match(Set mem (StoreL mem zero));
 7869   predicate(!needs_releasing_store(n));
 7870 
 7871   ins_cost(INSN_COST);
 7872   format %{ "str  zr, $mem\t# int" %}
 7873 
 7874   ins_encode(aarch64_enc_str0(mem));
 7875 
 7876   ins_pipe(istore_mem);
 7877 %}
 7878 
 7879 // Store Pointer
 7880 instruct storeP(iRegP src, memory8 mem)
 7881 %{
 7882   match(Set mem (StoreP mem src));
 7883   predicate(!needs_releasing_store(n));
 7884 
 7885   ins_cost(INSN_COST);
 7886   format %{ "str  $src, $mem\t# ptr" %}
 7887 
 7888   ins_encode(aarch64_enc_str(src, mem));
 7889 
 7890   ins_pipe(istore_reg_mem);
 7891 %}
 7892 
 7893 // Store Pointer
 7894 instruct storeimmP0(immP0 zero, memory8 mem)
 7895 %{
 7896   match(Set mem (StoreP mem zero));
 7897   predicate(!needs_releasing_store(n));
 7898 
 7899   ins_cost(INSN_COST);
 7900   format %{ "str zr, $mem\t# ptr" %}
 7901 
 7902   ins_encode(aarch64_enc_str0(mem));
 7903 
 7904   ins_pipe(istore_mem);
 7905 %}
 7906 
 7907 // Store Compressed Pointer
 7908 instruct storeN(iRegN src, memory4 mem)
 7909 %{
 7910   match(Set mem (StoreN mem src));
 7911   predicate(!needs_releasing_store(n));
 7912 
 7913   ins_cost(INSN_COST);
 7914   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7915 
 7916   ins_encode(aarch64_enc_strw(src, mem));
 7917 
 7918   ins_pipe(istore_reg_mem);
 7919 %}
 7920 
 7921 instruct storeImmN0(immN0 zero, memory4 mem)
 7922 %{
 7923   match(Set mem (StoreN mem zero));
 7924   predicate(!needs_releasing_store(n));
 7925 
 7926   ins_cost(INSN_COST);
 7927   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7928 
 7929   ins_encode(aarch64_enc_strw0(mem));
 7930 
 7931   ins_pipe(istore_mem);
 7932 %}
 7933 
 7934 // Store Float
 7935 instruct storeF(vRegF src, memory4 mem)
 7936 %{
 7937   match(Set mem (StoreF mem src));
 7938   predicate(!needs_releasing_store(n));
 7939 
 7940   ins_cost(INSN_COST);
 7941   format %{ "strs  $src, $mem\t# float" %}
 7942 
 7943   ins_encode( aarch64_enc_strs(src, mem) );
 7944 
 7945   ins_pipe(pipe_class_memory);
 7946 %}
 7947 
 7948 // TODO
 7949 // implement storeImmF0 and storeFImmPacked
 7950 
 7951 // Store Double
 7952 instruct storeD(vRegD src, memory8 mem)
 7953 %{
 7954   match(Set mem (StoreD mem src));
 7955   predicate(!needs_releasing_store(n));
 7956 
 7957   ins_cost(INSN_COST);
 7958   format %{ "strd  $src, $mem\t# double" %}
 7959 
 7960   ins_encode( aarch64_enc_strd(src, mem) );
 7961 
 7962   ins_pipe(pipe_class_memory);
 7963 %}
 7964 
 7965 // Store Compressed Klass Pointer
 7966 instruct storeNKlass(iRegN src, memory4 mem)
 7967 %{
 7968   predicate(!needs_releasing_store(n));
 7969   match(Set mem (StoreNKlass mem src));
 7970 
 7971   ins_cost(INSN_COST);
 7972   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7973 
 7974   ins_encode(aarch64_enc_strw(src, mem));
 7975 
 7976   ins_pipe(istore_reg_mem);
 7977 %}
 7978 
 7979 // TODO
 7980 // implement storeImmD0 and storeDImmPacked
 7981 
 7982 // prefetch instructions
 7983 // Must be safe to execute with invalid address (cannot fault).
 7984 
 7985 instruct prefetchalloc( memory8 mem ) %{
 7986   match(PrefetchAllocation mem);
 7987 
 7988   ins_cost(INSN_COST);
 7989   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7990 
 7991   ins_encode( aarch64_enc_prefetchw(mem) );
 7992 
 7993   ins_pipe(iload_prefetch);
 7994 %}
 7995 
 7996 //  ---------------- volatile loads and stores ----------------
 7997 
 7998 // Load Byte (8 bit signed)
 7999 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8000 %{
 8001   match(Set dst (LoadB mem));
 8002 
 8003   ins_cost(VOLATILE_REF_COST);
 8004   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8005 
 8006   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8007 
 8008   ins_pipe(pipe_serial);
 8009 %}
 8010 
 8011 // Load Byte (8 bit signed) into long
 8012 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8013 %{
 8014   match(Set dst (ConvI2L (LoadB mem)));
 8015 
 8016   ins_cost(VOLATILE_REF_COST);
 8017   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8018 
 8019   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8020 
 8021   ins_pipe(pipe_serial);
 8022 %}
 8023 
 8024 // Load Byte (8 bit unsigned)
 8025 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8026 %{
 8027   match(Set dst (LoadUB mem));
 8028 
 8029   ins_cost(VOLATILE_REF_COST);
 8030   format %{ "ldarb  $dst, $mem\t# byte" %}
 8031 
 8032   ins_encode(aarch64_enc_ldarb(dst, mem));
 8033 
 8034   ins_pipe(pipe_serial);
 8035 %}
 8036 
 8037 // Load Byte (8 bit unsigned) into long
 8038 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8039 %{
 8040   match(Set dst (ConvI2L (LoadUB mem)));
 8041 
 8042   ins_cost(VOLATILE_REF_COST);
 8043   format %{ "ldarb  $dst, $mem\t# byte" %}
 8044 
 8045   ins_encode(aarch64_enc_ldarb(dst, mem));
 8046 
 8047   ins_pipe(pipe_serial);
 8048 %}
 8049 
 8050 // Load Short (16 bit signed)
 8051 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8052 %{
 8053   match(Set dst (LoadS mem));
 8054 
 8055   ins_cost(VOLATILE_REF_COST);
 8056   format %{ "ldarshw  $dst, $mem\t# short" %}
 8057 
 8058   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8059 
 8060   ins_pipe(pipe_serial);
 8061 %}
 8062 
 8063 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8064 %{
 8065   match(Set dst (LoadUS mem));
 8066 
 8067   ins_cost(VOLATILE_REF_COST);
 8068   format %{ "ldarhw  $dst, $mem\t# short" %}
 8069 
 8070   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8071 
 8072   ins_pipe(pipe_serial);
 8073 %}
 8074 
 8075 // Load Short/Char (16 bit unsigned) into long
 8076 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8077 %{
 8078   match(Set dst (ConvI2L (LoadUS mem)));
 8079 
 8080   ins_cost(VOLATILE_REF_COST);
 8081   format %{ "ldarh  $dst, $mem\t# short" %}
 8082 
 8083   ins_encode(aarch64_enc_ldarh(dst, mem));
 8084 
 8085   ins_pipe(pipe_serial);
 8086 %}
 8087 
 8088 // Load Short/Char (16 bit signed) into long
 8089 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8090 %{
 8091   match(Set dst (ConvI2L (LoadS mem)));
 8092 
 8093   ins_cost(VOLATILE_REF_COST);
 8094   format %{ "ldarh  $dst, $mem\t# short" %}
 8095 
 8096   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8097 
 8098   ins_pipe(pipe_serial);
 8099 %}
 8100 
 8101 // Load Integer (32 bit signed)
 8102 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8103 %{
 8104   match(Set dst (LoadI mem));
 8105 
 8106   ins_cost(VOLATILE_REF_COST);
 8107   format %{ "ldarw  $dst, $mem\t# int" %}
 8108 
 8109   ins_encode(aarch64_enc_ldarw(dst, mem));
 8110 
 8111   ins_pipe(pipe_serial);
 8112 %}
 8113 
 8114 // Load Integer (32 bit unsigned) into long
 8115 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8116 %{
 8117   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8118 
 8119   ins_cost(VOLATILE_REF_COST);
 8120   format %{ "ldarw  $dst, $mem\t# int" %}
 8121 
 8122   ins_encode(aarch64_enc_ldarw(dst, mem));
 8123 
 8124   ins_pipe(pipe_serial);
 8125 %}
 8126 
 8127 // Load Long (64 bit signed)
 8128 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8129 %{
 8130   match(Set dst (LoadL mem));
 8131 
 8132   ins_cost(VOLATILE_REF_COST);
 8133   format %{ "ldar  $dst, $mem\t# int" %}
 8134 
 8135   ins_encode(aarch64_enc_ldar(dst, mem));
 8136 
 8137   ins_pipe(pipe_serial);
 8138 %}
 8139 
 8140 // Load Pointer
 8141 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8142 %{
 8143   match(Set dst (LoadP mem));
 8144   predicate(n->as_Load()->barrier_data() == 0);
 8145 
 8146   ins_cost(VOLATILE_REF_COST);
 8147   format %{ "ldar  $dst, $mem\t# ptr" %}
 8148 
 8149   ins_encode(aarch64_enc_ldar(dst, mem));
 8150 
 8151   ins_pipe(pipe_serial);
 8152 %}
 8153 
 8154 // Load Compressed Pointer
 8155 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8156 %{
 8157   match(Set dst (LoadN mem));
 8158 
 8159   ins_cost(VOLATILE_REF_COST);
 8160   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8161 
 8162   ins_encode(aarch64_enc_ldarw(dst, mem));
 8163 
 8164   ins_pipe(pipe_serial);
 8165 %}
 8166 
 8167 // Load Float
 8168 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8169 %{
 8170   match(Set dst (LoadF mem));
 8171 
 8172   ins_cost(VOLATILE_REF_COST);
 8173   format %{ "ldars  $dst, $mem\t# float" %}
 8174 
 8175   ins_encode( aarch64_enc_fldars(dst, mem) );
 8176 
 8177   ins_pipe(pipe_serial);
 8178 %}
 8179 
 8180 // Load Double
 8181 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8182 %{
 8183   match(Set dst (LoadD mem));
 8184 
 8185   ins_cost(VOLATILE_REF_COST);
 8186   format %{ "ldard  $dst, $mem\t# double" %}
 8187 
 8188   ins_encode( aarch64_enc_fldard(dst, mem) );
 8189 
 8190   ins_pipe(pipe_serial);
 8191 %}
 8192 
 8193 // Store Byte
 8194 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8195 %{
 8196   match(Set mem (StoreB mem src));
 8197 
 8198   ins_cost(VOLATILE_REF_COST);
 8199   format %{ "stlrb  $src, $mem\t# byte" %}
 8200 
 8201   ins_encode(aarch64_enc_stlrb(src, mem));
 8202 
 8203   ins_pipe(pipe_class_memory);
 8204 %}
 8205 
 8206 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8207 %{
 8208   match(Set mem (StoreB mem zero));
 8209 
 8210   ins_cost(VOLATILE_REF_COST);
 8211   format %{ "stlrb  zr, $mem\t# byte" %}
 8212 
 8213   ins_encode(aarch64_enc_stlrb0(mem));
 8214 
 8215   ins_pipe(pipe_class_memory);
 8216 %}
 8217 
 8218 // Store Char/Short
 8219 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8220 %{
 8221   match(Set mem (StoreC mem src));
 8222 
 8223   ins_cost(VOLATILE_REF_COST);
 8224   format %{ "stlrh  $src, $mem\t# short" %}
 8225 
 8226   ins_encode(aarch64_enc_stlrh(src, mem));
 8227 
 8228   ins_pipe(pipe_class_memory);
 8229 %}
 8230 
 8231 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8232 %{
 8233   match(Set mem (StoreC mem zero));
 8234 
 8235   ins_cost(VOLATILE_REF_COST);
 8236   format %{ "stlrh  zr, $mem\t# short" %}
 8237 
 8238   ins_encode(aarch64_enc_stlrh0(mem));
 8239 
 8240   ins_pipe(pipe_class_memory);
 8241 %}
 8242 
 8243 // Store Integer
 8244 
 8245 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8246 %{
 8247   match(Set mem(StoreI mem src));
 8248 
 8249   ins_cost(VOLATILE_REF_COST);
 8250   format %{ "stlrw  $src, $mem\t# int" %}
 8251 
 8252   ins_encode(aarch64_enc_stlrw(src, mem));
 8253 
 8254   ins_pipe(pipe_class_memory);
 8255 %}
 8256 
 8257 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8258 %{
 8259   match(Set mem(StoreI mem zero));
 8260 
 8261   ins_cost(VOLATILE_REF_COST);
 8262   format %{ "stlrw  zr, $mem\t# int" %}
 8263 
 8264   ins_encode(aarch64_enc_stlrw0(mem));
 8265 
 8266   ins_pipe(pipe_class_memory);
 8267 %}
 8268 
 8269 // Store Long (64 bit signed)
 8270 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8271 %{
 8272   match(Set mem (StoreL mem src));
 8273 
 8274   ins_cost(VOLATILE_REF_COST);
 8275   format %{ "stlr  $src, $mem\t# int" %}
 8276 
 8277   ins_encode(aarch64_enc_stlr(src, mem));
 8278 
 8279   ins_pipe(pipe_class_memory);
 8280 %}
 8281 
 8282 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8283 %{
 8284   match(Set mem (StoreL mem zero));
 8285 
 8286   ins_cost(VOLATILE_REF_COST);
 8287   format %{ "stlr  zr, $mem\t# int" %}
 8288 
 8289   ins_encode(aarch64_enc_stlr0(mem));
 8290 
 8291   ins_pipe(pipe_class_memory);
 8292 %}
 8293 
 8294 // Store Pointer
 8295 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8296 %{
 8297   match(Set mem (StoreP mem src));
 8298 
 8299   ins_cost(VOLATILE_REF_COST);
 8300   format %{ "stlr  $src, $mem\t# ptr" %}
 8301 
 8302   ins_encode(aarch64_enc_stlr(src, mem));
 8303 
 8304   ins_pipe(pipe_class_memory);
 8305 %}
 8306 
 8307 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8308 %{
 8309   match(Set mem (StoreP mem zero));
 8310 
 8311   ins_cost(VOLATILE_REF_COST);
 8312   format %{ "stlr  zr, $mem\t# ptr" %}
 8313 
 8314   ins_encode(aarch64_enc_stlr0(mem));
 8315 
 8316   ins_pipe(pipe_class_memory);
 8317 %}
 8318 
 8319 // Store Compressed Pointer
 8320 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8321 %{
 8322   match(Set mem (StoreN mem src));
 8323 
 8324   ins_cost(VOLATILE_REF_COST);
 8325   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8326 
 8327   ins_encode(aarch64_enc_stlrw(src, mem));
 8328 
 8329   ins_pipe(pipe_class_memory);
 8330 %}
 8331 
 8332 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8333 %{
 8334   match(Set mem (StoreN mem zero));
 8335 
 8336   ins_cost(VOLATILE_REF_COST);
 8337   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8338 
 8339   ins_encode(aarch64_enc_stlrw0(mem));
 8340 
 8341   ins_pipe(pipe_class_memory);
 8342 %}
 8343 
 8344 // Store Float
 8345 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8346 %{
 8347   match(Set mem (StoreF mem src));
 8348 
 8349   ins_cost(VOLATILE_REF_COST);
 8350   format %{ "stlrs  $src, $mem\t# float" %}
 8351 
 8352   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8353 
 8354   ins_pipe(pipe_class_memory);
 8355 %}
 8356 
 8357 // TODO
 8358 // implement storeImmF0 and storeFImmPacked
 8359 
 8360 // Store Double
 8361 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8362 %{
 8363   match(Set mem (StoreD mem src));
 8364 
 8365   ins_cost(VOLATILE_REF_COST);
 8366   format %{ "stlrd  $src, $mem\t# double" %}
 8367 
 8368   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8369 
 8370   ins_pipe(pipe_class_memory);
 8371 %}
 8372 
 8373 //  ---------------- end of volatile loads and stores ----------------
 8374 
 8375 instruct cacheWB(indirect addr)
 8376 %{
 8377   predicate(VM_Version::supports_data_cache_line_flush());
 8378   match(CacheWB addr);
 8379 
 8380   ins_cost(100);
 8381   format %{"cache wb $addr" %}
 8382   ins_encode %{
 8383     assert($addr->index_position() < 0, "should be");
 8384     assert($addr$$disp == 0, "should be");
 8385     __ cache_wb(Address($addr$$base$$Register, 0));
 8386   %}
 8387   ins_pipe(pipe_slow); // XXX
 8388 %}
 8389 
 8390 instruct cacheWBPreSync()
 8391 %{
 8392   predicate(VM_Version::supports_data_cache_line_flush());
 8393   match(CacheWBPreSync);
 8394 
 8395   ins_cost(100);
 8396   format %{"cache wb presync" %}
 8397   ins_encode %{
 8398     __ cache_wbsync(true);
 8399   %}
 8400   ins_pipe(pipe_slow); // XXX
 8401 %}
 8402 
 8403 instruct cacheWBPostSync()
 8404 %{
 8405   predicate(VM_Version::supports_data_cache_line_flush());
 8406   match(CacheWBPostSync);
 8407 
 8408   ins_cost(100);
 8409   format %{"cache wb postsync" %}
 8410   ins_encode %{
 8411     __ cache_wbsync(false);
 8412   %}
 8413   ins_pipe(pipe_slow); // XXX
 8414 %}
 8415 
 8416 // ============================================================================
 8417 // BSWAP Instructions
 8418 
 8419 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8420   match(Set dst (ReverseBytesI src));
 8421 
 8422   ins_cost(INSN_COST);
 8423   format %{ "revw  $dst, $src" %}
 8424 
 8425   ins_encode %{
 8426     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8427   %}
 8428 
 8429   ins_pipe(ialu_reg);
 8430 %}
 8431 
 8432 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8433   match(Set dst (ReverseBytesL src));
 8434 
 8435   ins_cost(INSN_COST);
 8436   format %{ "rev  $dst, $src" %}
 8437 
 8438   ins_encode %{
 8439     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8440   %}
 8441 
 8442   ins_pipe(ialu_reg);
 8443 %}
 8444 
 8445 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8446   match(Set dst (ReverseBytesUS src));
 8447 
 8448   ins_cost(INSN_COST);
 8449   format %{ "rev16w  $dst, $src" %}
 8450 
 8451   ins_encode %{
 8452     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8453   %}
 8454 
 8455   ins_pipe(ialu_reg);
 8456 %}
 8457 
 8458 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8459   match(Set dst (ReverseBytesS src));
 8460 
 8461   ins_cost(INSN_COST);
 8462   format %{ "rev16w  $dst, $src\n\t"
 8463             "sbfmw $dst, $dst, #0, #15" %}
 8464 
 8465   ins_encode %{
 8466     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8467     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8468   %}
 8469 
 8470   ins_pipe(ialu_reg);
 8471 %}
 8472 
 8473 // ============================================================================
 8474 // Zero Count Instructions
 8475 
 8476 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8477   match(Set dst (CountLeadingZerosI src));
 8478 
 8479   ins_cost(INSN_COST);
 8480   format %{ "clzw  $dst, $src" %}
 8481   ins_encode %{
 8482     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8483   %}
 8484 
 8485   ins_pipe(ialu_reg);
 8486 %}
 8487 
 8488 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8489   match(Set dst (CountLeadingZerosL src));
 8490 
 8491   ins_cost(INSN_COST);
 8492   format %{ "clz   $dst, $src" %}
 8493   ins_encode %{
 8494     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8495   %}
 8496 
 8497   ins_pipe(ialu_reg);
 8498 %}
 8499 
 8500 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8501   match(Set dst (CountTrailingZerosI src));
 8502 
 8503   ins_cost(INSN_COST * 2);
 8504   format %{ "rbitw  $dst, $src\n\t"
 8505             "clzw   $dst, $dst" %}
 8506   ins_encode %{
 8507     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8508     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8509   %}
 8510 
 8511   ins_pipe(ialu_reg);
 8512 %}
 8513 
 8514 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8515   match(Set dst (CountTrailingZerosL src));
 8516 
 8517   ins_cost(INSN_COST * 2);
 8518   format %{ "rbit   $dst, $src\n\t"
 8519             "clz    $dst, $dst" %}
 8520   ins_encode %{
 8521     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8522     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8523   %}
 8524 
 8525   ins_pipe(ialu_reg);
 8526 %}
 8527 
 8528 //---------- Population Count Instructions -------------------------------------
 8529 //
 8530 
 8531 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8532   predicate(UsePopCountInstruction);
 8533   match(Set dst (PopCountI src));
 8534   effect(TEMP tmp);
 8535   ins_cost(INSN_COST * 13);
 8536 
 8537   format %{ "movw   $src, $src\n\t"
 8538             "mov    $tmp, $src\t# vector (1D)\n\t"
 8539             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8540             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8541             "mov    $dst, $tmp\t# vector (1D)" %}
 8542   ins_encode %{
 8543     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8544     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8545     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8546     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8547     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8548   %}
 8549 
 8550   ins_pipe(pipe_class_default);
 8551 %}
 8552 
 8553 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8554   predicate(UsePopCountInstruction);
 8555   match(Set dst (PopCountI (LoadI mem)));
 8556   effect(TEMP tmp);
 8557   ins_cost(INSN_COST * 13);
 8558 
 8559   format %{ "ldrs   $tmp, $mem\n\t"
 8560             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8561             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8562             "mov    $dst, $tmp\t# vector (1D)" %}
 8563   ins_encode %{
 8564     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8565     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8566               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8567     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8568     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8569     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8570   %}
 8571 
 8572   ins_pipe(pipe_class_default);
 8573 %}
 8574 
 8575 // Note: Long.bitCount(long) returns an int.
 8576 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8577   predicate(UsePopCountInstruction);
 8578   match(Set dst (PopCountL src));
 8579   effect(TEMP tmp);
 8580   ins_cost(INSN_COST * 13);
 8581 
 8582   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8583             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8584             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8585             "mov    $dst, $tmp\t# vector (1D)" %}
 8586   ins_encode %{
 8587     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8588     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8589     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8590     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8591   %}
 8592 
 8593   ins_pipe(pipe_class_default);
 8594 %}
 8595 
 8596 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8597   predicate(UsePopCountInstruction);
 8598   match(Set dst (PopCountL (LoadL mem)));
 8599   effect(TEMP tmp);
 8600   ins_cost(INSN_COST * 13);
 8601 
 8602   format %{ "ldrd   $tmp, $mem\n\t"
 8603             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8604             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8605             "mov    $dst, $tmp\t# vector (1D)" %}
 8606   ins_encode %{
 8607     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8608     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8609               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8610     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8611     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8612     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8613   %}
 8614 
 8615   ins_pipe(pipe_class_default);
 8616 %}
 8617 
 8618 // ============================================================================
 8619 // MemBar Instruction
 8620 
 8621 instruct load_fence() %{
 8622   match(LoadFence);
 8623   ins_cost(VOLATILE_REF_COST);
 8624 
 8625   format %{ "load_fence" %}
 8626 
 8627   ins_encode %{
 8628     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8629   %}
 8630   ins_pipe(pipe_serial);
 8631 %}
 8632 
 8633 instruct unnecessary_membar_acquire() %{
 8634   predicate(unnecessary_acquire(n));
 8635   match(MemBarAcquire);
 8636   ins_cost(0);
 8637 
 8638   format %{ "membar_acquire (elided)" %}
 8639 
 8640   ins_encode %{
 8641     __ block_comment("membar_acquire (elided)");
 8642   %}
 8643 
 8644   ins_pipe(pipe_class_empty);
 8645 %}
 8646 
 8647 instruct membar_acquire() %{
 8648   match(MemBarAcquire);
 8649   ins_cost(VOLATILE_REF_COST);
 8650 
 8651   format %{ "membar_acquire\n\t"
 8652             "dmb ish" %}
 8653 
 8654   ins_encode %{
 8655     __ block_comment("membar_acquire");
 8656     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8657   %}
 8658 
 8659   ins_pipe(pipe_serial);
 8660 %}
 8661 
 8662 
 8663 instruct membar_acquire_lock() %{
 8664   match(MemBarAcquireLock);
 8665   ins_cost(VOLATILE_REF_COST);
 8666 
 8667   format %{ "membar_acquire_lock (elided)" %}
 8668 
 8669   ins_encode %{
 8670     __ block_comment("membar_acquire_lock (elided)");
 8671   %}
 8672 
 8673   ins_pipe(pipe_serial);
 8674 %}
 8675 
 8676 instruct store_fence() %{
 8677   match(StoreFence);
 8678   ins_cost(VOLATILE_REF_COST);
 8679 
 8680   format %{ "store_fence" %}
 8681 
 8682   ins_encode %{
 8683     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8684   %}
 8685   ins_pipe(pipe_serial);
 8686 %}
 8687 
 8688 instruct unnecessary_membar_release() %{
 8689   predicate(unnecessary_release(n));
 8690   match(MemBarRelease);
 8691   ins_cost(0);
 8692 
 8693   format %{ "membar_release (elided)" %}
 8694 
 8695   ins_encode %{
 8696     __ block_comment("membar_release (elided)");
 8697   %}
 8698   ins_pipe(pipe_serial);
 8699 %}
 8700 
 8701 instruct membar_release() %{
 8702   match(MemBarRelease);
 8703   ins_cost(VOLATILE_REF_COST);
 8704 
 8705   format %{ "membar_release\n\t"
 8706             "dmb ish" %}
 8707 
 8708   ins_encode %{
 8709     __ block_comment("membar_release");
 8710     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8711   %}
 8712   ins_pipe(pipe_serial);
 8713 %}
 8714 
 8715 instruct membar_storestore() %{
 8716   match(MemBarStoreStore);
 8717   ins_cost(VOLATILE_REF_COST);
 8718 
 8719   format %{ "MEMBAR-store-store" %}
 8720 
 8721   ins_encode %{
 8722     __ membar(Assembler::StoreStore);
 8723   %}
 8724   ins_pipe(pipe_serial);
 8725 %}
 8726 
 8727 instruct membar_release_lock() %{
 8728   match(MemBarReleaseLock);
 8729   ins_cost(VOLATILE_REF_COST);
 8730 
 8731   format %{ "membar_release_lock (elided)" %}
 8732 
 8733   ins_encode %{
 8734     __ block_comment("membar_release_lock (elided)");
 8735   %}
 8736 
 8737   ins_pipe(pipe_serial);
 8738 %}
 8739 
 8740 instruct unnecessary_membar_volatile() %{
 8741   predicate(unnecessary_volatile(n));
 8742   match(MemBarVolatile);
 8743   ins_cost(0);
 8744 
 8745   format %{ "membar_volatile (elided)" %}
 8746 
 8747   ins_encode %{
 8748     __ block_comment("membar_volatile (elided)");
 8749   %}
 8750 
 8751   ins_pipe(pipe_serial);
 8752 %}
 8753 
 8754 instruct membar_volatile() %{
 8755   match(MemBarVolatile);
 8756   ins_cost(VOLATILE_REF_COST*100);
 8757 
 8758   format %{ "membar_volatile\n\t"
 8759              "dmb ish"%}
 8760 
 8761   ins_encode %{
 8762     __ block_comment("membar_volatile");
 8763     __ membar(Assembler::StoreLoad);
 8764   %}
 8765 
 8766   ins_pipe(pipe_serial);
 8767 %}
 8768 
 8769 // ============================================================================
 8770 // Cast/Convert Instructions
 8771 
 8772 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8773   match(Set dst (CastX2P src));
 8774 
 8775   ins_cost(INSN_COST);
 8776   format %{ "mov $dst, $src\t# long -> ptr" %}
 8777 
 8778   ins_encode %{
 8779     if ($dst$$reg != $src$$reg) {
 8780       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8781     }
 8782   %}
 8783 
 8784   ins_pipe(ialu_reg);
 8785 %}
 8786 
 8787 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8788   match(Set dst (CastP2X src));
 8789 
 8790   ins_cost(INSN_COST);
 8791   format %{ "mov $dst, $src\t# ptr -> long" %}
 8792 
 8793   ins_encode %{
 8794     if ($dst$$reg != $src$$reg) {
 8795       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8796     }
 8797   %}
 8798 
 8799   ins_pipe(ialu_reg);
 8800 %}
 8801 
 8802 // Convert oop into int for vectors alignment masking
 8803 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8804   match(Set dst (ConvL2I (CastP2X src)));
 8805 
 8806   ins_cost(INSN_COST);
 8807   format %{ "movw $dst, $src\t# ptr -> int" %}
 8808   ins_encode %{
 8809     __ movw($dst$$Register, $src$$Register);
 8810   %}
 8811 
 8812   ins_pipe(ialu_reg);
 8813 %}
 8814 
 8815 // Convert compressed oop into int for vectors alignment masking
 8816 // in case of 32bit oops (heap < 4Gb).
 8817 instruct convN2I(iRegINoSp dst, iRegN src)
 8818 %{
 8819   predicate(CompressedOops::shift() == 0);
 8820   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8821 
 8822   ins_cost(INSN_COST);
 8823   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8824   ins_encode %{
 8825     __ movw($dst$$Register, $src$$Register);
 8826   %}
 8827 
 8828   ins_pipe(ialu_reg);
 8829 %}
 8830 
 8831 
 8832 // Convert oop pointer into compressed form
 8833 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8834   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8835   match(Set dst (EncodeP src));
 8836   effect(KILL cr);
 8837   ins_cost(INSN_COST * 3);
 8838   format %{ "encode_heap_oop $dst, $src" %}
 8839   ins_encode %{
 8840     Register s = $src$$Register;
 8841     Register d = $dst$$Register;
 8842     __ encode_heap_oop(d, s);
 8843   %}
 8844   ins_pipe(ialu_reg);
 8845 %}
 8846 
 8847 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8848   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8849   match(Set dst (EncodeP src));
 8850   ins_cost(INSN_COST * 3);
 8851   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8852   ins_encode %{
 8853     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8854   %}
 8855   ins_pipe(ialu_reg);
 8856 %}
 8857 
 8858 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8859   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8860             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8861   match(Set dst (DecodeN src));
 8862   ins_cost(INSN_COST * 3);
 8863   format %{ "decode_heap_oop $dst, $src" %}
 8864   ins_encode %{
 8865     Register s = $src$$Register;
 8866     Register d = $dst$$Register;
 8867     __ decode_heap_oop(d, s);
 8868   %}
 8869   ins_pipe(ialu_reg);
 8870 %}
 8871 
 8872 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8873   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8874             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8875   match(Set dst (DecodeN src));
 8876   ins_cost(INSN_COST * 3);
 8877   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8878   ins_encode %{
 8879     Register s = $src$$Register;
 8880     Register d = $dst$$Register;
 8881     __ decode_heap_oop_not_null(d, s);
 8882   %}
 8883   ins_pipe(ialu_reg);
 8884 %}
 8885 
 8886 // n.b. AArch64 implementations of encode_klass_not_null and
 8887 // decode_klass_not_null do not modify the flags register so, unlike
 8888 // Intel, we don't kill CR as a side effect here
 8889 
 8890 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8891   match(Set dst (EncodePKlass src));
 8892 
 8893   ins_cost(INSN_COST * 3);
 8894   format %{ "encode_klass_not_null $dst,$src" %}
 8895 
 8896   ins_encode %{
 8897     Register src_reg = as_Register($src$$reg);
 8898     Register dst_reg = as_Register($dst$$reg);
 8899     __ encode_klass_not_null(dst_reg, src_reg);
 8900   %}
 8901 
 8902    ins_pipe(ialu_reg);
 8903 %}
 8904 
 8905 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8906   match(Set dst (DecodeNKlass src));
 8907 
 8908   ins_cost(INSN_COST * 3);
 8909   format %{ "decode_klass_not_null $dst,$src" %}
 8910 
 8911   ins_encode %{
 8912     Register src_reg = as_Register($src$$reg);
 8913     Register dst_reg = as_Register($dst$$reg);
 8914     if (dst_reg != src_reg) {
 8915       __ decode_klass_not_null(dst_reg, src_reg);
 8916     } else {
 8917       __ decode_klass_not_null(dst_reg);
 8918     }
 8919   %}
 8920 
 8921    ins_pipe(ialu_reg);
 8922 %}
 8923 
 8924 instruct checkCastPP(iRegPNoSp dst)
 8925 %{
 8926   match(Set dst (CheckCastPP dst));
 8927 
 8928   size(0);
 8929   format %{ "# checkcastPP of $dst" %}
 8930   ins_encode(/* empty encoding */);
 8931   ins_pipe(pipe_class_empty);
 8932 %}
 8933 
 8934 instruct castPP(iRegPNoSp dst)
 8935 %{
 8936   match(Set dst (CastPP dst));
 8937 
 8938   size(0);
 8939   format %{ "# castPP of $dst" %}
 8940   ins_encode(/* empty encoding */);
 8941   ins_pipe(pipe_class_empty);
 8942 %}
 8943 
 8944 instruct castII(iRegI dst)
 8945 %{
 8946   match(Set dst (CastII dst));
 8947 
 8948   size(0);
 8949   format %{ "# castII of $dst" %}
 8950   ins_encode(/* empty encoding */);
 8951   ins_cost(0);
 8952   ins_pipe(pipe_class_empty);
 8953 %}
 8954 
 8955 instruct castLL(iRegL dst)
 8956 %{
 8957   match(Set dst (CastLL dst));
 8958 
 8959   size(0);
 8960   format %{ "# castLL of $dst" %}
 8961   ins_encode(/* empty encoding */);
 8962   ins_cost(0);
 8963   ins_pipe(pipe_class_empty);
 8964 %}
 8965 
 8966 instruct castFF(vRegF dst)
 8967 %{
 8968   match(Set dst (CastFF dst));
 8969 
 8970   size(0);
 8971   format %{ "# castFF of $dst" %}
 8972   ins_encode(/* empty encoding */);
 8973   ins_cost(0);
 8974   ins_pipe(pipe_class_empty);
 8975 %}
 8976 
 8977 instruct castDD(vRegD dst)
 8978 %{
 8979   match(Set dst (CastDD dst));
 8980 
 8981   size(0);
 8982   format %{ "# castDD of $dst" %}
 8983   ins_encode(/* empty encoding */);
 8984   ins_cost(0);
 8985   ins_pipe(pipe_class_empty);
 8986 %}
 8987 
 8988 instruct castVVD(vecD dst)
 8989 %{
 8990   match(Set dst (CastVV dst));
 8991 
 8992   size(0);
 8993   format %{ "# castVV of $dst" %}
 8994   ins_encode(/* empty encoding */);
 8995   ins_cost(0);
 8996   ins_pipe(pipe_class_empty);
 8997 %}
 8998 
 8999 instruct castVVX(vecX dst)
 9000 %{
 9001   match(Set dst (CastVV dst));
 9002 
 9003   size(0);
 9004   format %{ "# castVV of $dst" %}
 9005   ins_encode(/* empty encoding */);
 9006   ins_cost(0);
 9007   ins_pipe(pipe_class_empty);
 9008 %}
 9009 
 9010 instruct castVV(vReg dst)
 9011 %{
 9012   match(Set dst (CastVV dst));
 9013 
 9014   size(0);
 9015   format %{ "# castVV of $dst" %}
 9016   ins_encode(/* empty encoding */);
 9017   ins_cost(0);
 9018   ins_pipe(pipe_class_empty);
 9019 %}
 9020 
 9021 // ============================================================================
 9022 // Atomic operation instructions
 9023 //
 9024 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9025 // Store{PIL}Conditional instructions using a normal load for the
 9026 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9027 //
 9028 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9029 // pair to lock object allocations from Eden space when not using
 9030 // TLABs.
 9031 //
 9032 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9033 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9034 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9035 // only for 64-bit.
 9036 //
 9037 // We implement LoadPLocked and StorePLocked instructions using,
 9038 // respectively the AArch64 hw load-exclusive and store-conditional
 9039 // instructions. Whereas we must implement each of
 9040 // Store{IL}Conditional using a CAS which employs a pair of
 9041 // instructions comprising a load-exclusive followed by a
 9042 // store-conditional.
 9043 
 9044 
 9045 // Locked-load (linked load) of the current heap-top
 9046 // used when updating the eden heap top
 9047 // implemented using ldaxr on AArch64
 9048 
 9049 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9050 %{
 9051   match(Set dst (LoadPLocked mem));
 9052 
 9053   ins_cost(VOLATILE_REF_COST);
 9054 
 9055   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9056 
 9057   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9058 
 9059   ins_pipe(pipe_serial);
 9060 %}
 9061 
 9062 // Conditional-store of the updated heap-top.
 9063 // Used during allocation of the shared heap.
 9064 // Sets flag (EQ) on success.
 9065 // implemented using stlxr on AArch64.
 9066 
 9067 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9068 %{
 9069   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9070 
 9071   ins_cost(VOLATILE_REF_COST);
 9072 
 9073  // TODO
 9074  // do we need to do a store-conditional release or can we just use a
 9075  // plain store-conditional?
 9076 
 9077   format %{
 9078     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9079     "cmpw rscratch1, zr\t# EQ on successful write"
 9080   %}
 9081 
 9082   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9083 
 9084   ins_pipe(pipe_serial);
 9085 %}
 9086 
 9087 
 9088 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
 9089 // when attempting to rebias a lock towards the current thread.  We
 9090 // must use the acquire form of cmpxchg in order to guarantee acquire
 9091 // semantics in this case.
 9092 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9093 %{
 9094   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9095 
 9096   ins_cost(VOLATILE_REF_COST);
 9097 
 9098   format %{
 9099     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9100     "cmpw rscratch1, zr\t# EQ on successful write"
 9101   %}
 9102 
 9103   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9104 
 9105   ins_pipe(pipe_slow);
 9106 %}
 9107 
 9108 // storeIConditional also has acquire semantics, for no better reason
 9109 // than matching storeLConditional.  At the time of writing this
 9110 // comment storeIConditional was not used anywhere by AArch64.
 9111 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9112 %{
 9113   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9114 
 9115   ins_cost(VOLATILE_REF_COST);
 9116 
 9117   format %{
 9118     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9119     "cmpw rscratch1, zr\t# EQ on successful write"
 9120   %}
 9121 
 9122   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9123 
 9124   ins_pipe(pipe_slow);
 9125 %}
 9126 
 9127 // standard CompareAndSwapX when we are using barriers
 9128 // these have higher priority than the rules selected by a predicate
 9129 
 9130 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9131 // can't match them
 9132 
 9133 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9134 
 9135   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9136   ins_cost(2 * VOLATILE_REF_COST);
 9137 
 9138   effect(KILL cr);
 9139 
 9140   format %{
 9141     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9142     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9143   %}
 9144 
 9145   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9146             aarch64_enc_cset_eq(res));
 9147 
 9148   ins_pipe(pipe_slow);
 9149 %}
 9150 
 9151 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9152 
 9153   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9154   ins_cost(2 * VOLATILE_REF_COST);
 9155 
 9156   effect(KILL cr);
 9157 
 9158   format %{
 9159     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9160     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9161   %}
 9162 
 9163   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9164             aarch64_enc_cset_eq(res));
 9165 
 9166   ins_pipe(pipe_slow);
 9167 %}
 9168 
 9169 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9170 
 9171   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9172   ins_cost(2 * VOLATILE_REF_COST);
 9173 
 9174   effect(KILL cr);
 9175 
 9176  format %{
 9177     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9178     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9179  %}
 9180 
 9181  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9182             aarch64_enc_cset_eq(res));
 9183 
 9184   ins_pipe(pipe_slow);
 9185 %}
 9186 
 9187 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9188 
 9189   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9190   ins_cost(2 * VOLATILE_REF_COST);
 9191 
 9192   effect(KILL cr);
 9193 
 9194  format %{
 9195     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9196     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9197  %}
 9198 
 9199  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9200             aarch64_enc_cset_eq(res));
 9201 
 9202   ins_pipe(pipe_slow);
 9203 %}
 9204 
 9205 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9206 
 9207   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9208   predicate(n->as_LoadStore()->barrier_data() == 0);
 9209   ins_cost(2 * VOLATILE_REF_COST);
 9210 
 9211   effect(KILL cr);
 9212 
 9213  format %{
 9214     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9215     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9216  %}
 9217 
 9218  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9219             aarch64_enc_cset_eq(res));
 9220 
 9221   ins_pipe(pipe_slow);
 9222 %}
 9223 
 9224 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9225 
 9226   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9227   ins_cost(2 * VOLATILE_REF_COST);
 9228 
 9229   effect(KILL cr);
 9230 
 9231  format %{
 9232     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9233     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9234  %}
 9235 
 9236  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9237             aarch64_enc_cset_eq(res));
 9238 
 9239   ins_pipe(pipe_slow);
 9240 %}
 9241 
 9242 // alternative CompareAndSwapX when we are eliding barriers
 9243 
 9244 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9245 
 9246   predicate(needs_acquiring_load_exclusive(n));
 9247   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9248   ins_cost(VOLATILE_REF_COST);
 9249 
 9250   effect(KILL cr);
 9251 
 9252   format %{
 9253     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9254     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9255   %}
 9256 
 9257   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9258             aarch64_enc_cset_eq(res));
 9259 
 9260   ins_pipe(pipe_slow);
 9261 %}
 9262 
 9263 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9264 
 9265   predicate(needs_acquiring_load_exclusive(n));
 9266   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9267   ins_cost(VOLATILE_REF_COST);
 9268 
 9269   effect(KILL cr);
 9270 
 9271   format %{
 9272     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9273     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9274   %}
 9275 
 9276   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9277             aarch64_enc_cset_eq(res));
 9278 
 9279   ins_pipe(pipe_slow);
 9280 %}
 9281 
 9282 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9283 
 9284   predicate(needs_acquiring_load_exclusive(n));
 9285   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9286   ins_cost(VOLATILE_REF_COST);
 9287 
 9288   effect(KILL cr);
 9289 
 9290  format %{
 9291     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9292     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9293  %}
 9294 
 9295  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9296             aarch64_enc_cset_eq(res));
 9297 
 9298   ins_pipe(pipe_slow);
 9299 %}
 9300 
 9301 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9302 
 9303   predicate(needs_acquiring_load_exclusive(n));
 9304   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9305   ins_cost(VOLATILE_REF_COST);
 9306 
 9307   effect(KILL cr);
 9308 
 9309  format %{
 9310     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9311     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9312  %}
 9313 
 9314  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9315             aarch64_enc_cset_eq(res));
 9316 
 9317   ins_pipe(pipe_slow);
 9318 %}
 9319 
 9320 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9321 
 9322   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9323   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9324   ins_cost(VOLATILE_REF_COST);
 9325 
 9326   effect(KILL cr);
 9327 
 9328  format %{
 9329     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9330     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9331  %}
 9332 
 9333  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9334             aarch64_enc_cset_eq(res));
 9335 
 9336   ins_pipe(pipe_slow);
 9337 %}
 9338 
 9339 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9340 
 9341   predicate(needs_acquiring_load_exclusive(n));
 9342   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9343   ins_cost(VOLATILE_REF_COST);
 9344 
 9345   effect(KILL cr);
 9346 
 9347  format %{
 9348     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9349     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9350  %}
 9351 
 9352  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9353             aarch64_enc_cset_eq(res));
 9354 
 9355   ins_pipe(pipe_slow);
 9356 %}
 9357 
 9358 
 9359 // ---------------------------------------------------------------------
 9360 
 9361 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9362 
 9363 // Sundry CAS operations.  Note that release is always true,
 9364 // regardless of the memory ordering of the CAS.  This is because we
 9365 // need the volatile case to be sequentially consistent but there is
 9366 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9367 // can't check the type of memory ordering here, so we always emit a
 9368 // STLXR.
 9369 
 9370 // This section is generated from aarch64_ad_cas.m4
 9371 
 9372 
 9373 
 9374 // This pattern is generated automatically from cas.m4.
 9375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9376 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9377 
 9378   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9379   ins_cost(2 * VOLATILE_REF_COST);
 9380   effect(TEMP_DEF res, KILL cr);
 9381   format %{
 9382     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9383   %}
 9384   ins_encode %{
 9385     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9386                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9387                /*weak*/ false, $res$$Register);
 9388     __ sxtbw($res$$Register, $res$$Register);
 9389   %}
 9390   ins_pipe(pipe_slow);
 9391 %}
 9392 
 9393 // This pattern is generated automatically from cas.m4.
 9394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9395 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9396 
 9397   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9398   ins_cost(2 * VOLATILE_REF_COST);
 9399   effect(TEMP_DEF res, KILL cr);
 9400   format %{
 9401     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9402   %}
 9403   ins_encode %{
 9404     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9405                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9406                /*weak*/ false, $res$$Register);
 9407     __ sxthw($res$$Register, $res$$Register);
 9408   %}
 9409   ins_pipe(pipe_slow);
 9410 %}
 9411 
 9412 // This pattern is generated automatically from cas.m4.
 9413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9414 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9415 
 9416   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9417   ins_cost(2 * VOLATILE_REF_COST);
 9418   effect(TEMP_DEF res, KILL cr);
 9419   format %{
 9420     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9421   %}
 9422   ins_encode %{
 9423     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9424                Assembler::word, /*acquire*/ false, /*release*/ true,
 9425                /*weak*/ false, $res$$Register);
 9426   %}
 9427   ins_pipe(pipe_slow);
 9428 %}
 9429 
 9430 // This pattern is generated automatically from cas.m4.
 9431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9432 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9433 
 9434   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9435   ins_cost(2 * VOLATILE_REF_COST);
 9436   effect(TEMP_DEF res, KILL cr);
 9437   format %{
 9438     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9439   %}
 9440   ins_encode %{
 9441     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9442                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9443                /*weak*/ false, $res$$Register);
 9444   %}
 9445   ins_pipe(pipe_slow);
 9446 %}
 9447 
 9448 // This pattern is generated automatically from cas.m4.
 9449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9450 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9451 
 9452   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9453   ins_cost(2 * VOLATILE_REF_COST);
 9454   effect(TEMP_DEF res, KILL cr);
 9455   format %{
 9456     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9457   %}
 9458   ins_encode %{
 9459     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9460                Assembler::word, /*acquire*/ false, /*release*/ true,
 9461                /*weak*/ false, $res$$Register);
 9462   %}
 9463   ins_pipe(pipe_slow);
 9464 %}
 9465 
 9466 // This pattern is generated automatically from cas.m4.
 9467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9468 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9469   predicate(n->as_LoadStore()->barrier_data() == 0);
 9470   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9471   ins_cost(2 * VOLATILE_REF_COST);
 9472   effect(TEMP_DEF res, KILL cr);
 9473   format %{
 9474     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9475   %}
 9476   ins_encode %{
 9477     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9478                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9479                /*weak*/ false, $res$$Register);
 9480   %}
 9481   ins_pipe(pipe_slow);
 9482 %}
 9483 
 9484 // This pattern is generated automatically from cas.m4.
 9485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9486 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9487   predicate(needs_acquiring_load_exclusive(n));
 9488   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9489   ins_cost(VOLATILE_REF_COST);
 9490   effect(TEMP_DEF res, KILL cr);
 9491   format %{
 9492     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9493   %}
 9494   ins_encode %{
 9495     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9496                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9497                /*weak*/ false, $res$$Register);
 9498     __ sxtbw($res$$Register, $res$$Register);
 9499   %}
 9500   ins_pipe(pipe_slow);
 9501 %}
 9502 
 9503 // This pattern is generated automatically from cas.m4.
 9504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9505 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9506   predicate(needs_acquiring_load_exclusive(n));
 9507   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9508   ins_cost(VOLATILE_REF_COST);
 9509   effect(TEMP_DEF res, KILL cr);
 9510   format %{
 9511     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9512   %}
 9513   ins_encode %{
 9514     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9515                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9516                /*weak*/ false, $res$$Register);
 9517     __ sxthw($res$$Register, $res$$Register);
 9518   %}
 9519   ins_pipe(pipe_slow);
 9520 %}
 9521 
 9522 // This pattern is generated automatically from cas.m4.
 9523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9524 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9525   predicate(needs_acquiring_load_exclusive(n));
 9526   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9527   ins_cost(VOLATILE_REF_COST);
 9528   effect(TEMP_DEF res, KILL cr);
 9529   format %{
 9530     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9531   %}
 9532   ins_encode %{
 9533     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9534                Assembler::word, /*acquire*/ true, /*release*/ true,
 9535                /*weak*/ false, $res$$Register);
 9536   %}
 9537   ins_pipe(pipe_slow);
 9538 %}
 9539 
 9540 // This pattern is generated automatically from cas.m4.
 9541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9542 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9543   predicate(needs_acquiring_load_exclusive(n));
 9544   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9545   ins_cost(VOLATILE_REF_COST);
 9546   effect(TEMP_DEF res, KILL cr);
 9547   format %{
 9548     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9549   %}
 9550   ins_encode %{
 9551     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9552                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9553                /*weak*/ false, $res$$Register);
 9554   %}
 9555   ins_pipe(pipe_slow);
 9556 %}
 9557 
 9558 // This pattern is generated automatically from cas.m4.
 9559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9560 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9561   predicate(needs_acquiring_load_exclusive(n));
 9562   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9563   ins_cost(VOLATILE_REF_COST);
 9564   effect(TEMP_DEF res, KILL cr);
 9565   format %{
 9566     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9567   %}
 9568   ins_encode %{
 9569     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9570                Assembler::word, /*acquire*/ true, /*release*/ true,
 9571                /*weak*/ false, $res$$Register);
 9572   %}
 9573   ins_pipe(pipe_slow);
 9574 %}
 9575 
 9576 // This pattern is generated automatically from cas.m4.
 9577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9578 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9579   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9580   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9581   ins_cost(VOLATILE_REF_COST);
 9582   effect(TEMP_DEF res, KILL cr);
 9583   format %{
 9584     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9585   %}
 9586   ins_encode %{
 9587     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9588                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9589                /*weak*/ false, $res$$Register);
 9590   %}
 9591   ins_pipe(pipe_slow);
 9592 %}
 9593 
 9594 // This pattern is generated automatically from cas.m4.
 9595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9596 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9597 
 9598   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9599   ins_cost(2 * VOLATILE_REF_COST);
 9600   effect(KILL cr);
 9601   format %{
 9602     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9603     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9604   %}
 9605   ins_encode %{
 9606     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9607                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9608                /*weak*/ true, noreg);
 9609     __ csetw($res$$Register, Assembler::EQ);
 9610   %}
 9611   ins_pipe(pipe_slow);
 9612 %}
 9613 
 9614 // This pattern is generated automatically from cas.m4.
 9615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9616 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9617 
 9618   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9619   ins_cost(2 * VOLATILE_REF_COST);
 9620   effect(KILL cr);
 9621   format %{
 9622     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9623     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9624   %}
 9625   ins_encode %{
 9626     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9627                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9628                /*weak*/ true, noreg);
 9629     __ csetw($res$$Register, Assembler::EQ);
 9630   %}
 9631   ins_pipe(pipe_slow);
 9632 %}
 9633 
 9634 // This pattern is generated automatically from cas.m4.
 9635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9636 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9637 
 9638   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9639   ins_cost(2 * VOLATILE_REF_COST);
 9640   effect(KILL cr);
 9641   format %{
 9642     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9643     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9644   %}
 9645   ins_encode %{
 9646     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9647                Assembler::word, /*acquire*/ false, /*release*/ true,
 9648                /*weak*/ true, noreg);
 9649     __ csetw($res$$Register, Assembler::EQ);
 9650   %}
 9651   ins_pipe(pipe_slow);
 9652 %}
 9653 
 9654 // This pattern is generated automatically from cas.m4.
 9655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9656 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9657 
 9658   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9659   ins_cost(2 * VOLATILE_REF_COST);
 9660   effect(KILL cr);
 9661   format %{
 9662     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9663     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9664   %}
 9665   ins_encode %{
 9666     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9667                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9668                /*weak*/ true, noreg);
 9669     __ csetw($res$$Register, Assembler::EQ);
 9670   %}
 9671   ins_pipe(pipe_slow);
 9672 %}
 9673 
 9674 // This pattern is generated automatically from cas.m4.
 9675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9676 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9677 
 9678   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9679   ins_cost(2 * VOLATILE_REF_COST);
 9680   effect(KILL cr);
 9681   format %{
 9682     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9683     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9684   %}
 9685   ins_encode %{
 9686     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9687                Assembler::word, /*acquire*/ false, /*release*/ true,
 9688                /*weak*/ true, noreg);
 9689     __ csetw($res$$Register, Assembler::EQ);
 9690   %}
 9691   ins_pipe(pipe_slow);
 9692 %}
 9693 
 9694 // This pattern is generated automatically from cas.m4.
 9695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9696 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9697   predicate(n->as_LoadStore()->barrier_data() == 0);
 9698   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9699   ins_cost(2 * VOLATILE_REF_COST);
 9700   effect(KILL cr);
 9701   format %{
 9702     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9703     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9704   %}
 9705   ins_encode %{
 9706     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9707                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9708                /*weak*/ true, noreg);
 9709     __ csetw($res$$Register, Assembler::EQ);
 9710   %}
 9711   ins_pipe(pipe_slow);
 9712 %}
 9713 
 9714 // This pattern is generated automatically from cas.m4.
 9715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9716 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9717   predicate(needs_acquiring_load_exclusive(n));
 9718   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9719   ins_cost(VOLATILE_REF_COST);
 9720   effect(KILL cr);
 9721   format %{
 9722     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9723     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9724   %}
 9725   ins_encode %{
 9726     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9727                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9728                /*weak*/ true, noreg);
 9729     __ csetw($res$$Register, Assembler::EQ);
 9730   %}
 9731   ins_pipe(pipe_slow);
 9732 %}
 9733 
 9734 // This pattern is generated automatically from cas.m4.
 9735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9736 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9737   predicate(needs_acquiring_load_exclusive(n));
 9738   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9739   ins_cost(VOLATILE_REF_COST);
 9740   effect(KILL cr);
 9741   format %{
 9742     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9743     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9744   %}
 9745   ins_encode %{
 9746     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9747                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9748                /*weak*/ true, noreg);
 9749     __ csetw($res$$Register, Assembler::EQ);
 9750   %}
 9751   ins_pipe(pipe_slow);
 9752 %}
 9753 
 9754 // This pattern is generated automatically from cas.m4.
 9755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9756 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9757   predicate(needs_acquiring_load_exclusive(n));
 9758   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9759   ins_cost(VOLATILE_REF_COST);
 9760   effect(KILL cr);
 9761   format %{
 9762     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9763     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9764   %}
 9765   ins_encode %{
 9766     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9767                Assembler::word, /*acquire*/ true, /*release*/ true,
 9768                /*weak*/ true, noreg);
 9769     __ csetw($res$$Register, Assembler::EQ);
 9770   %}
 9771   ins_pipe(pipe_slow);
 9772 %}
 9773 
 9774 // This pattern is generated automatically from cas.m4.
 9775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9776 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9777   predicate(needs_acquiring_load_exclusive(n));
 9778   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9779   ins_cost(VOLATILE_REF_COST);
 9780   effect(KILL cr);
 9781   format %{
 9782     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9783     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9784   %}
 9785   ins_encode %{
 9786     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9787                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9788                /*weak*/ true, noreg);
 9789     __ csetw($res$$Register, Assembler::EQ);
 9790   %}
 9791   ins_pipe(pipe_slow);
 9792 %}
 9793 
 9794 // This pattern is generated automatically from cas.m4.
 9795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9796 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9797   predicate(needs_acquiring_load_exclusive(n));
 9798   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9799   ins_cost(VOLATILE_REF_COST);
 9800   effect(KILL cr);
 9801   format %{
 9802     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9803     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9804   %}
 9805   ins_encode %{
 9806     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9807                Assembler::word, /*acquire*/ true, /*release*/ true,
 9808                /*weak*/ true, noreg);
 9809     __ csetw($res$$Register, Assembler::EQ);
 9810   %}
 9811   ins_pipe(pipe_slow);
 9812 %}
 9813 
 9814 // This pattern is generated automatically from cas.m4.
 9815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9816 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9817   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9818   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9819   ins_cost(VOLATILE_REF_COST);
 9820   effect(KILL cr);
 9821   format %{
 9822     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9823     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9824   %}
 9825   ins_encode %{
 9826     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9827                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9828                /*weak*/ true, noreg);
 9829     __ csetw($res$$Register, Assembler::EQ);
 9830   %}
 9831   ins_pipe(pipe_slow);
 9832 %}
 9833 
 9834 // END This section of the file is automatically generated. Do not edit --------------
 9835 // ---------------------------------------------------------------------
 9836 
 9837 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9838   match(Set prev (GetAndSetI mem newv));
 9839   ins_cost(2 * VOLATILE_REF_COST);
 9840   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9841   ins_encode %{
 9842     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9843   %}
 9844   ins_pipe(pipe_serial);
 9845 %}
 9846 
 9847 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9848   match(Set prev (GetAndSetL mem newv));
 9849   ins_cost(2 * VOLATILE_REF_COST);
 9850   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9851   ins_encode %{
 9852     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9853   %}
 9854   ins_pipe(pipe_serial);
 9855 %}
 9856 
 9857 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9858   match(Set prev (GetAndSetN mem newv));
 9859   ins_cost(2 * VOLATILE_REF_COST);
 9860   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9861   ins_encode %{
 9862     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9863   %}
 9864   ins_pipe(pipe_serial);
 9865 %}
 9866 
 9867 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9868   predicate(n->as_LoadStore()->barrier_data() == 0);
 9869   match(Set prev (GetAndSetP mem newv));
 9870   ins_cost(2 * VOLATILE_REF_COST);
 9871   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9872   ins_encode %{
 9873     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9874   %}
 9875   ins_pipe(pipe_serial);
 9876 %}
 9877 
 9878 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9879   predicate(needs_acquiring_load_exclusive(n));
 9880   match(Set prev (GetAndSetI mem newv));
 9881   ins_cost(VOLATILE_REF_COST);
 9882   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9883   ins_encode %{
 9884     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9885   %}
 9886   ins_pipe(pipe_serial);
 9887 %}
 9888 
 9889 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9890   predicate(needs_acquiring_load_exclusive(n));
 9891   match(Set prev (GetAndSetL mem newv));
 9892   ins_cost(VOLATILE_REF_COST);
 9893   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9894   ins_encode %{
 9895     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9896   %}
 9897   ins_pipe(pipe_serial);
 9898 %}
 9899 
 9900 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9901   predicate(needs_acquiring_load_exclusive(n));
 9902   match(Set prev (GetAndSetN mem newv));
 9903   ins_cost(VOLATILE_REF_COST);
 9904   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9905   ins_encode %{
 9906     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9907   %}
 9908   ins_pipe(pipe_serial);
 9909 %}
 9910 
 9911 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9912   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9913   match(Set prev (GetAndSetP mem newv));
 9914   ins_cost(VOLATILE_REF_COST);
 9915   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9916   ins_encode %{
 9917     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9918   %}
 9919   ins_pipe(pipe_serial);
 9920 %}
 9921 
 9922 
 9923 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9924   match(Set newval (GetAndAddL mem incr));
 9925   ins_cost(2 * VOLATILE_REF_COST + 1);
 9926   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9927   ins_encode %{
 9928     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9929   %}
 9930   ins_pipe(pipe_serial);
 9931 %}
 9932 
 9933 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9934   predicate(n->as_LoadStore()->result_not_used());
 9935   match(Set dummy (GetAndAddL mem incr));
 9936   ins_cost(2 * VOLATILE_REF_COST);
 9937   format %{ "get_and_addL [$mem], $incr" %}
 9938   ins_encode %{
 9939     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9940   %}
 9941   ins_pipe(pipe_serial);
 9942 %}
 9943 
 9944 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9945   match(Set newval (GetAndAddL mem incr));
 9946   ins_cost(2 * VOLATILE_REF_COST + 1);
 9947   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9948   ins_encode %{
 9949     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9950   %}
 9951   ins_pipe(pipe_serial);
 9952 %}
 9953 
 9954 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9955   predicate(n->as_LoadStore()->result_not_used());
 9956   match(Set dummy (GetAndAddL mem incr));
 9957   ins_cost(2 * VOLATILE_REF_COST);
 9958   format %{ "get_and_addL [$mem], $incr" %}
 9959   ins_encode %{
 9960     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9961   %}
 9962   ins_pipe(pipe_serial);
 9963 %}
 9964 
 9965 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9966   match(Set newval (GetAndAddI mem incr));
 9967   ins_cost(2 * VOLATILE_REF_COST + 1);
 9968   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9969   ins_encode %{
 9970     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9971   %}
 9972   ins_pipe(pipe_serial);
 9973 %}
 9974 
 9975 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9976   predicate(n->as_LoadStore()->result_not_used());
 9977   match(Set dummy (GetAndAddI mem incr));
 9978   ins_cost(2 * VOLATILE_REF_COST);
 9979   format %{ "get_and_addI [$mem], $incr" %}
 9980   ins_encode %{
 9981     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9982   %}
 9983   ins_pipe(pipe_serial);
 9984 %}
 9985 
 9986 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9987   match(Set newval (GetAndAddI mem incr));
 9988   ins_cost(2 * VOLATILE_REF_COST + 1);
 9989   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9990   ins_encode %{
 9991     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9992   %}
 9993   ins_pipe(pipe_serial);
 9994 %}
 9995 
 9996 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9997   predicate(n->as_LoadStore()->result_not_used());
 9998   match(Set dummy (GetAndAddI mem incr));
 9999   ins_cost(2 * VOLATILE_REF_COST);
10000   format %{ "get_and_addI [$mem], $incr" %}
10001   ins_encode %{
10002     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10003   %}
10004   ins_pipe(pipe_serial);
10005 %}
10006 
10007 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10008   predicate(needs_acquiring_load_exclusive(n));
10009   match(Set newval (GetAndAddL mem incr));
10010   ins_cost(VOLATILE_REF_COST + 1);
10011   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10012   ins_encode %{
10013     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10014   %}
10015   ins_pipe(pipe_serial);
10016 %}
10017 
10018 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10019   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10020   match(Set dummy (GetAndAddL mem incr));
10021   ins_cost(VOLATILE_REF_COST);
10022   format %{ "get_and_addL_acq [$mem], $incr" %}
10023   ins_encode %{
10024     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10025   %}
10026   ins_pipe(pipe_serial);
10027 %}
10028 
10029 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10030   predicate(needs_acquiring_load_exclusive(n));
10031   match(Set newval (GetAndAddL mem incr));
10032   ins_cost(VOLATILE_REF_COST + 1);
10033   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10034   ins_encode %{
10035     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10036   %}
10037   ins_pipe(pipe_serial);
10038 %}
10039 
10040 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10041   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10042   match(Set dummy (GetAndAddL mem incr));
10043   ins_cost(VOLATILE_REF_COST);
10044   format %{ "get_and_addL_acq [$mem], $incr" %}
10045   ins_encode %{
10046     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10047   %}
10048   ins_pipe(pipe_serial);
10049 %}
10050 
10051 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10052   predicate(needs_acquiring_load_exclusive(n));
10053   match(Set newval (GetAndAddI mem incr));
10054   ins_cost(VOLATILE_REF_COST + 1);
10055   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10056   ins_encode %{
10057     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10058   %}
10059   ins_pipe(pipe_serial);
10060 %}
10061 
10062 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10063   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10064   match(Set dummy (GetAndAddI mem incr));
10065   ins_cost(VOLATILE_REF_COST);
10066   format %{ "get_and_addI_acq [$mem], $incr" %}
10067   ins_encode %{
10068     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10069   %}
10070   ins_pipe(pipe_serial);
10071 %}
10072 
10073 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10074   predicate(needs_acquiring_load_exclusive(n));
10075   match(Set newval (GetAndAddI mem incr));
10076   ins_cost(VOLATILE_REF_COST + 1);
10077   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10078   ins_encode %{
10079     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10080   %}
10081   ins_pipe(pipe_serial);
10082 %}
10083 
10084 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10085   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10086   match(Set dummy (GetAndAddI mem incr));
10087   ins_cost(VOLATILE_REF_COST);
10088   format %{ "get_and_addI_acq [$mem], $incr" %}
10089   ins_encode %{
10090     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10091   %}
10092   ins_pipe(pipe_serial);
10093 %}
10094 
10095 // Manifest a CmpL result in an integer register.
10096 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10097 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10098 %{
10099   match(Set dst (CmpL3 src1 src2));
10100   effect(KILL flags);
10101 
10102   ins_cost(INSN_COST * 6);
10103   format %{
10104       "cmp $src1, $src2"
10105       "csetw $dst, ne"
10106       "cnegw $dst, lt"
10107   %}
10108   // format %{ "CmpL3 $dst, $src1, $src2" %}
10109   ins_encode %{
10110     __ cmp($src1$$Register, $src2$$Register);
10111     __ csetw($dst$$Register, Assembler::NE);
10112     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10113   %}
10114 
10115   ins_pipe(pipe_class_default);
10116 %}
10117 
10118 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10119 %{
10120   match(Set dst (CmpL3 src1 src2));
10121   effect(KILL flags);
10122 
10123   ins_cost(INSN_COST * 6);
10124   format %{
10125       "cmp $src1, $src2"
10126       "csetw $dst, ne"
10127       "cnegw $dst, lt"
10128   %}
10129   ins_encode %{
10130     int32_t con = (int32_t)$src2$$constant;
10131      if (con < 0) {
10132       __ adds(zr, $src1$$Register, -con);
10133     } else {
10134       __ subs(zr, $src1$$Register, con);
10135     }
10136     __ csetw($dst$$Register, Assembler::NE);
10137     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10138   %}
10139 
10140   ins_pipe(pipe_class_default);
10141 %}
10142 
10143 // ============================================================================
10144 // Conditional Move Instructions
10145 
10146 // n.b. we have identical rules for both a signed compare op (cmpOp)
10147 // and an unsigned compare op (cmpOpU). it would be nice if we could
10148 // define an op class which merged both inputs and use it to type the
10149 // argument to a single rule. unfortunatelyt his fails because the
10150 // opclass does not live up to the COND_INTER interface of its
10151 // component operands. When the generic code tries to negate the
10152 // operand it ends up running the generci Machoper::negate method
10153 // which throws a ShouldNotHappen. So, we have to provide two flavours
10154 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10155 
10156 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10157   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10158 
10159   ins_cost(INSN_COST * 2);
10160   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10161 
10162   ins_encode %{
10163     __ cselw(as_Register($dst$$reg),
10164              as_Register($src2$$reg),
10165              as_Register($src1$$reg),
10166              (Assembler::Condition)$cmp$$cmpcode);
10167   %}
10168 
10169   ins_pipe(icond_reg_reg);
10170 %}
10171 
10172 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10173   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10174 
10175   ins_cost(INSN_COST * 2);
10176   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10177 
10178   ins_encode %{
10179     __ cselw(as_Register($dst$$reg),
10180              as_Register($src2$$reg),
10181              as_Register($src1$$reg),
10182              (Assembler::Condition)$cmp$$cmpcode);
10183   %}
10184 
10185   ins_pipe(icond_reg_reg);
10186 %}
10187 
10188 // special cases where one arg is zero
10189 
10190 // n.b. this is selected in preference to the rule above because it
10191 // avoids loading constant 0 into a source register
10192 
10193 // TODO
10194 // we ought only to be able to cull one of these variants as the ideal
10195 // transforms ought always to order the zero consistently (to left/right?)
10196 
10197 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10198   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10199 
10200   ins_cost(INSN_COST * 2);
10201   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10202 
10203   ins_encode %{
10204     __ cselw(as_Register($dst$$reg),
10205              as_Register($src$$reg),
10206              zr,
10207              (Assembler::Condition)$cmp$$cmpcode);
10208   %}
10209 
10210   ins_pipe(icond_reg);
10211 %}
10212 
10213 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10214   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10215 
10216   ins_cost(INSN_COST * 2);
10217   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10218 
10219   ins_encode %{
10220     __ cselw(as_Register($dst$$reg),
10221              as_Register($src$$reg),
10222              zr,
10223              (Assembler::Condition)$cmp$$cmpcode);
10224   %}
10225 
10226   ins_pipe(icond_reg);
10227 %}
10228 
10229 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10230   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10231 
10232   ins_cost(INSN_COST * 2);
10233   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10234 
10235   ins_encode %{
10236     __ cselw(as_Register($dst$$reg),
10237              zr,
10238              as_Register($src$$reg),
10239              (Assembler::Condition)$cmp$$cmpcode);
10240   %}
10241 
10242   ins_pipe(icond_reg);
10243 %}
10244 
10245 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10246   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10247 
10248   ins_cost(INSN_COST * 2);
10249   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10250 
10251   ins_encode %{
10252     __ cselw(as_Register($dst$$reg),
10253              zr,
10254              as_Register($src$$reg),
10255              (Assembler::Condition)$cmp$$cmpcode);
10256   %}
10257 
10258   ins_pipe(icond_reg);
10259 %}
10260 
10261 // special case for creating a boolean 0 or 1
10262 
10263 // n.b. this is selected in preference to the rule above because it
10264 // avoids loading constants 0 and 1 into a source register
10265 
10266 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10267   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10268 
10269   ins_cost(INSN_COST * 2);
10270   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10271 
10272   ins_encode %{
10273     // equivalently
10274     // cset(as_Register($dst$$reg),
10275     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10276     __ csincw(as_Register($dst$$reg),
10277              zr,
10278              zr,
10279              (Assembler::Condition)$cmp$$cmpcode);
10280   %}
10281 
10282   ins_pipe(icond_none);
10283 %}
10284 
10285 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10286   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10287 
10288   ins_cost(INSN_COST * 2);
10289   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10290 
10291   ins_encode %{
10292     // equivalently
10293     // cset(as_Register($dst$$reg),
10294     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10295     __ csincw(as_Register($dst$$reg),
10296              zr,
10297              zr,
10298              (Assembler::Condition)$cmp$$cmpcode);
10299   %}
10300 
10301   ins_pipe(icond_none);
10302 %}
10303 
10304 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10305   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10306 
10307   ins_cost(INSN_COST * 2);
10308   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10309 
10310   ins_encode %{
10311     __ csel(as_Register($dst$$reg),
10312             as_Register($src2$$reg),
10313             as_Register($src1$$reg),
10314             (Assembler::Condition)$cmp$$cmpcode);
10315   %}
10316 
10317   ins_pipe(icond_reg_reg);
10318 %}
10319 
10320 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10321   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10322 
10323   ins_cost(INSN_COST * 2);
10324   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10325 
10326   ins_encode %{
10327     __ csel(as_Register($dst$$reg),
10328             as_Register($src2$$reg),
10329             as_Register($src1$$reg),
10330             (Assembler::Condition)$cmp$$cmpcode);
10331   %}
10332 
10333   ins_pipe(icond_reg_reg);
10334 %}
10335 
10336 // special cases where one arg is zero
10337 
10338 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10339   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10340 
10341   ins_cost(INSN_COST * 2);
10342   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10343 
10344   ins_encode %{
10345     __ csel(as_Register($dst$$reg),
10346             zr,
10347             as_Register($src$$reg),
10348             (Assembler::Condition)$cmp$$cmpcode);
10349   %}
10350 
10351   ins_pipe(icond_reg);
10352 %}
10353 
10354 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10355   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10356 
10357   ins_cost(INSN_COST * 2);
10358   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10359 
10360   ins_encode %{
10361     __ csel(as_Register($dst$$reg),
10362             zr,
10363             as_Register($src$$reg),
10364             (Assembler::Condition)$cmp$$cmpcode);
10365   %}
10366 
10367   ins_pipe(icond_reg);
10368 %}
10369 
10370 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10371   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10372 
10373   ins_cost(INSN_COST * 2);
10374   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10375 
10376   ins_encode %{
10377     __ csel(as_Register($dst$$reg),
10378             as_Register($src$$reg),
10379             zr,
10380             (Assembler::Condition)$cmp$$cmpcode);
10381   %}
10382 
10383   ins_pipe(icond_reg);
10384 %}
10385 
10386 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10387   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10388 
10389   ins_cost(INSN_COST * 2);
10390   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10391 
10392   ins_encode %{
10393     __ csel(as_Register($dst$$reg),
10394             as_Register($src$$reg),
10395             zr,
10396             (Assembler::Condition)$cmp$$cmpcode);
10397   %}
10398 
10399   ins_pipe(icond_reg);
10400 %}
10401 
10402 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10403   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10404 
10405   ins_cost(INSN_COST * 2);
10406   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10407 
10408   ins_encode %{
10409     __ csel(as_Register($dst$$reg),
10410             as_Register($src2$$reg),
10411             as_Register($src1$$reg),
10412             (Assembler::Condition)$cmp$$cmpcode);
10413   %}
10414 
10415   ins_pipe(icond_reg_reg);
10416 %}
10417 
10418 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10419   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10420 
10421   ins_cost(INSN_COST * 2);
10422   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10423 
10424   ins_encode %{
10425     __ csel(as_Register($dst$$reg),
10426             as_Register($src2$$reg),
10427             as_Register($src1$$reg),
10428             (Assembler::Condition)$cmp$$cmpcode);
10429   %}
10430 
10431   ins_pipe(icond_reg_reg);
10432 %}
10433 
10434 // special cases where one arg is zero
10435 
10436 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10437   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10438 
10439   ins_cost(INSN_COST * 2);
10440   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10441 
10442   ins_encode %{
10443     __ csel(as_Register($dst$$reg),
10444             zr,
10445             as_Register($src$$reg),
10446             (Assembler::Condition)$cmp$$cmpcode);
10447   %}
10448 
10449   ins_pipe(icond_reg);
10450 %}
10451 
10452 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10453   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10454 
10455   ins_cost(INSN_COST * 2);
10456   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10457 
10458   ins_encode %{
10459     __ csel(as_Register($dst$$reg),
10460             zr,
10461             as_Register($src$$reg),
10462             (Assembler::Condition)$cmp$$cmpcode);
10463   %}
10464 
10465   ins_pipe(icond_reg);
10466 %}
10467 
10468 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10469   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10470 
10471   ins_cost(INSN_COST * 2);
10472   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10473 
10474   ins_encode %{
10475     __ csel(as_Register($dst$$reg),
10476             as_Register($src$$reg),
10477             zr,
10478             (Assembler::Condition)$cmp$$cmpcode);
10479   %}
10480 
10481   ins_pipe(icond_reg);
10482 %}
10483 
10484 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10485   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10486 
10487   ins_cost(INSN_COST * 2);
10488   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10489 
10490   ins_encode %{
10491     __ csel(as_Register($dst$$reg),
10492             as_Register($src$$reg),
10493             zr,
10494             (Assembler::Condition)$cmp$$cmpcode);
10495   %}
10496 
10497   ins_pipe(icond_reg);
10498 %}
10499 
10500 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10501   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10502 
10503   ins_cost(INSN_COST * 2);
10504   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10505 
10506   ins_encode %{
10507     __ cselw(as_Register($dst$$reg),
10508              as_Register($src2$$reg),
10509              as_Register($src1$$reg),
10510              (Assembler::Condition)$cmp$$cmpcode);
10511   %}
10512 
10513   ins_pipe(icond_reg_reg);
10514 %}
10515 
10516 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10517   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10518 
10519   ins_cost(INSN_COST * 2);
10520   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10521 
10522   ins_encode %{
10523     __ cselw(as_Register($dst$$reg),
10524              as_Register($src2$$reg),
10525              as_Register($src1$$reg),
10526              (Assembler::Condition)$cmp$$cmpcode);
10527   %}
10528 
10529   ins_pipe(icond_reg_reg);
10530 %}
10531 
10532 // special cases where one arg is zero
10533 
10534 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10535   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10536 
10537   ins_cost(INSN_COST * 2);
10538   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10539 
10540   ins_encode %{
10541     __ cselw(as_Register($dst$$reg),
10542              zr,
10543              as_Register($src$$reg),
10544              (Assembler::Condition)$cmp$$cmpcode);
10545   %}
10546 
10547   ins_pipe(icond_reg);
10548 %}
10549 
10550 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10551   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10552 
10553   ins_cost(INSN_COST * 2);
10554   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10555 
10556   ins_encode %{
10557     __ cselw(as_Register($dst$$reg),
10558              zr,
10559              as_Register($src$$reg),
10560              (Assembler::Condition)$cmp$$cmpcode);
10561   %}
10562 
10563   ins_pipe(icond_reg);
10564 %}
10565 
10566 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10567   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10568 
10569   ins_cost(INSN_COST * 2);
10570   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10571 
10572   ins_encode %{
10573     __ cselw(as_Register($dst$$reg),
10574              as_Register($src$$reg),
10575              zr,
10576              (Assembler::Condition)$cmp$$cmpcode);
10577   %}
10578 
10579   ins_pipe(icond_reg);
10580 %}
10581 
10582 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10583   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10584 
10585   ins_cost(INSN_COST * 2);
10586   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10587 
10588   ins_encode %{
10589     __ cselw(as_Register($dst$$reg),
10590              as_Register($src$$reg),
10591              zr,
10592              (Assembler::Condition)$cmp$$cmpcode);
10593   %}
10594 
10595   ins_pipe(icond_reg);
10596 %}
10597 
10598 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10599 %{
10600   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10601 
10602   ins_cost(INSN_COST * 3);
10603 
10604   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10605   ins_encode %{
10606     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10607     __ fcsels(as_FloatRegister($dst$$reg),
10608               as_FloatRegister($src2$$reg),
10609               as_FloatRegister($src1$$reg),
10610               cond);
10611   %}
10612 
10613   ins_pipe(fp_cond_reg_reg_s);
10614 %}
10615 
10616 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10617 %{
10618   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10619 
10620   ins_cost(INSN_COST * 3);
10621 
10622   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10623   ins_encode %{
10624     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10625     __ fcsels(as_FloatRegister($dst$$reg),
10626               as_FloatRegister($src2$$reg),
10627               as_FloatRegister($src1$$reg),
10628               cond);
10629   %}
10630 
10631   ins_pipe(fp_cond_reg_reg_s);
10632 %}
10633 
10634 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10635 %{
10636   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10637 
10638   ins_cost(INSN_COST * 3);
10639 
10640   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10641   ins_encode %{
10642     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10643     __ fcseld(as_FloatRegister($dst$$reg),
10644               as_FloatRegister($src2$$reg),
10645               as_FloatRegister($src1$$reg),
10646               cond);
10647   %}
10648 
10649   ins_pipe(fp_cond_reg_reg_d);
10650 %}
10651 
10652 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10653 %{
10654   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10655 
10656   ins_cost(INSN_COST * 3);
10657 
10658   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10659   ins_encode %{
10660     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10661     __ fcseld(as_FloatRegister($dst$$reg),
10662               as_FloatRegister($src2$$reg),
10663               as_FloatRegister($src1$$reg),
10664               cond);
10665   %}
10666 
10667   ins_pipe(fp_cond_reg_reg_d);
10668 %}
10669 
10670 // ============================================================================
10671 // Arithmetic Instructions
10672 //
10673 
10674 // Integer Addition
10675 
10676 // TODO
10677 // these currently employ operations which do not set CR and hence are
10678 // not flagged as killing CR but we would like to isolate the cases
10679 // where we want to set flags from those where we don't. need to work
10680 // out how to do that.
10681 
10682 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10683   match(Set dst (AddI src1 src2));
10684 
10685   ins_cost(INSN_COST);
10686   format %{ "addw  $dst, $src1, $src2" %}
10687 
10688   ins_encode %{
10689     __ addw(as_Register($dst$$reg),
10690             as_Register($src1$$reg),
10691             as_Register($src2$$reg));
10692   %}
10693 
10694   ins_pipe(ialu_reg_reg);
10695 %}
10696 
10697 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10698   match(Set dst (AddI src1 src2));
10699 
10700   ins_cost(INSN_COST);
10701   format %{ "addw $dst, $src1, $src2" %}
10702 
10703   // use opcode to indicate that this is an add not a sub
10704   opcode(0x0);
10705 
10706   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10707 
10708   ins_pipe(ialu_reg_imm);
10709 %}
10710 
10711 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10712   match(Set dst (AddI (ConvL2I src1) src2));
10713 
10714   ins_cost(INSN_COST);
10715   format %{ "addw $dst, $src1, $src2" %}
10716 
10717   // use opcode to indicate that this is an add not a sub
10718   opcode(0x0);
10719 
10720   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10721 
10722   ins_pipe(ialu_reg_imm);
10723 %}
10724 
10725 // Pointer Addition
10726 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10727   match(Set dst (AddP src1 src2));
10728 
10729   ins_cost(INSN_COST);
10730   format %{ "add $dst, $src1, $src2\t# ptr" %}
10731 
10732   ins_encode %{
10733     __ add(as_Register($dst$$reg),
10734            as_Register($src1$$reg),
10735            as_Register($src2$$reg));
10736   %}
10737 
10738   ins_pipe(ialu_reg_reg);
10739 %}
10740 
10741 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10742   match(Set dst (AddP src1 (ConvI2L src2)));
10743 
10744   ins_cost(1.9 * INSN_COST);
10745   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10746 
10747   ins_encode %{
10748     __ add(as_Register($dst$$reg),
10749            as_Register($src1$$reg),
10750            as_Register($src2$$reg), ext::sxtw);
10751   %}
10752 
10753   ins_pipe(ialu_reg_reg);
10754 %}
10755 
10756 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10757   match(Set dst (AddP src1 (LShiftL src2 scale)));
10758 
10759   ins_cost(1.9 * INSN_COST);
10760   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10761 
10762   ins_encode %{
10763     __ lea(as_Register($dst$$reg),
10764            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10765                    Address::lsl($scale$$constant)));
10766   %}
10767 
10768   ins_pipe(ialu_reg_reg_shift);
10769 %}
10770 
10771 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10772   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10773 
10774   ins_cost(1.9 * INSN_COST);
10775   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10776 
10777   ins_encode %{
10778     __ lea(as_Register($dst$$reg),
10779            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10780                    Address::sxtw($scale$$constant)));
10781   %}
10782 
10783   ins_pipe(ialu_reg_reg_shift);
10784 %}
10785 
10786 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10787   match(Set dst (LShiftL (ConvI2L src) scale));
10788 
10789   ins_cost(INSN_COST);
10790   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10791 
10792   ins_encode %{
10793     __ sbfiz(as_Register($dst$$reg),
10794           as_Register($src$$reg),
10795           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10796   %}
10797 
10798   ins_pipe(ialu_reg_shift);
10799 %}
10800 
10801 // Pointer Immediate Addition
10802 // n.b. this needs to be more expensive than using an indirect memory
10803 // operand
10804 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10805   match(Set dst (AddP src1 src2));
10806 
10807   ins_cost(INSN_COST);
10808   format %{ "add $dst, $src1, $src2\t# ptr" %}
10809 
10810   // use opcode to indicate that this is an add not a sub
10811   opcode(0x0);
10812 
10813   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10814 
10815   ins_pipe(ialu_reg_imm);
10816 %}
10817 
10818 // Long Addition
10819 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10820 
10821   match(Set dst (AddL src1 src2));
10822 
10823   ins_cost(INSN_COST);
10824   format %{ "add  $dst, $src1, $src2" %}
10825 
10826   ins_encode %{
10827     __ add(as_Register($dst$$reg),
10828            as_Register($src1$$reg),
10829            as_Register($src2$$reg));
10830   %}
10831 
10832   ins_pipe(ialu_reg_reg);
10833 %}
10834 
10835 // No constant pool entries requiredLong Immediate Addition.
10836 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10837   match(Set dst (AddL src1 src2));
10838 
10839   ins_cost(INSN_COST);
10840   format %{ "add $dst, $src1, $src2" %}
10841 
10842   // use opcode to indicate that this is an add not a sub
10843   opcode(0x0);
10844 
10845   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10846 
10847   ins_pipe(ialu_reg_imm);
10848 %}
10849 
10850 // Integer Subtraction
10851 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10852   match(Set dst (SubI src1 src2));
10853 
10854   ins_cost(INSN_COST);
10855   format %{ "subw  $dst, $src1, $src2" %}
10856 
10857   ins_encode %{
10858     __ subw(as_Register($dst$$reg),
10859             as_Register($src1$$reg),
10860             as_Register($src2$$reg));
10861   %}
10862 
10863   ins_pipe(ialu_reg_reg);
10864 %}
10865 
10866 // Immediate Subtraction
10867 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10868   match(Set dst (SubI src1 src2));
10869 
10870   ins_cost(INSN_COST);
10871   format %{ "subw $dst, $src1, $src2" %}
10872 
10873   // use opcode to indicate that this is a sub not an add
10874   opcode(0x1);
10875 
10876   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10877 
10878   ins_pipe(ialu_reg_imm);
10879 %}
10880 
10881 // Long Subtraction
10882 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10883 
10884   match(Set dst (SubL src1 src2));
10885 
10886   ins_cost(INSN_COST);
10887   format %{ "sub  $dst, $src1, $src2" %}
10888 
10889   ins_encode %{
10890     __ sub(as_Register($dst$$reg),
10891            as_Register($src1$$reg),
10892            as_Register($src2$$reg));
10893   %}
10894 
10895   ins_pipe(ialu_reg_reg);
10896 %}
10897 
10898 // No constant pool entries requiredLong Immediate Subtraction.
10899 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10900   match(Set dst (SubL src1 src2));
10901 
10902   ins_cost(INSN_COST);
10903   format %{ "sub$dst, $src1, $src2" %}
10904 
10905   // use opcode to indicate that this is a sub not an add
10906   opcode(0x1);
10907 
10908   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10909 
10910   ins_pipe(ialu_reg_imm);
10911 %}
10912 
10913 // Integer Negation (special case for sub)
10914 
10915 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10916   match(Set dst (SubI zero src));
10917 
10918   ins_cost(INSN_COST);
10919   format %{ "negw $dst, $src\t# int" %}
10920 
10921   ins_encode %{
10922     __ negw(as_Register($dst$$reg),
10923             as_Register($src$$reg));
10924   %}
10925 
10926   ins_pipe(ialu_reg);
10927 %}
10928 
10929 // Long Negation
10930 
10931 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10932   match(Set dst (SubL zero src));
10933 
10934   ins_cost(INSN_COST);
10935   format %{ "neg $dst, $src\t# long" %}
10936 
10937   ins_encode %{
10938     __ neg(as_Register($dst$$reg),
10939            as_Register($src$$reg));
10940   %}
10941 
10942   ins_pipe(ialu_reg);
10943 %}
10944 
10945 // Integer Multiply
10946 
10947 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10948   match(Set dst (MulI src1 src2));
10949 
10950   ins_cost(INSN_COST * 3);
10951   format %{ "mulw  $dst, $src1, $src2" %}
10952 
10953   ins_encode %{
10954     __ mulw(as_Register($dst$$reg),
10955             as_Register($src1$$reg),
10956             as_Register($src2$$reg));
10957   %}
10958 
10959   ins_pipe(imul_reg_reg);
10960 %}
10961 
10962 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10963   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10964 
10965   ins_cost(INSN_COST * 3);
10966   format %{ "smull  $dst, $src1, $src2" %}
10967 
10968   ins_encode %{
10969     __ smull(as_Register($dst$$reg),
10970              as_Register($src1$$reg),
10971              as_Register($src2$$reg));
10972   %}
10973 
10974   ins_pipe(imul_reg_reg);
10975 %}
10976 
10977 // Long Multiply
10978 
10979 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10980   match(Set dst (MulL src1 src2));
10981 
10982   ins_cost(INSN_COST * 5);
10983   format %{ "mul  $dst, $src1, $src2" %}
10984 
10985   ins_encode %{
10986     __ mul(as_Register($dst$$reg),
10987            as_Register($src1$$reg),
10988            as_Register($src2$$reg));
10989   %}
10990 
10991   ins_pipe(lmul_reg_reg);
10992 %}
10993 
10994 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10995 %{
10996   match(Set dst (MulHiL src1 src2));
10997 
10998   ins_cost(INSN_COST * 7);
10999   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11000 
11001   ins_encode %{
11002     __ smulh(as_Register($dst$$reg),
11003              as_Register($src1$$reg),
11004              as_Register($src2$$reg));
11005   %}
11006 
11007   ins_pipe(lmul_reg_reg);
11008 %}
11009 
11010 // Combined Integer Multiply & Add/Sub
11011 
11012 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11013   match(Set dst (AddI src3 (MulI src1 src2)));
11014 
11015   ins_cost(INSN_COST * 3);
11016   format %{ "madd  $dst, $src1, $src2, $src3" %}
11017 
11018   ins_encode %{
11019     __ maddw(as_Register($dst$$reg),
11020              as_Register($src1$$reg),
11021              as_Register($src2$$reg),
11022              as_Register($src3$$reg));
11023   %}
11024 
11025   ins_pipe(imac_reg_reg);
11026 %}
11027 
11028 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11029   match(Set dst (SubI src3 (MulI src1 src2)));
11030 
11031   ins_cost(INSN_COST * 3);
11032   format %{ "msub  $dst, $src1, $src2, $src3" %}
11033 
11034   ins_encode %{
11035     __ msubw(as_Register($dst$$reg),
11036              as_Register($src1$$reg),
11037              as_Register($src2$$reg),
11038              as_Register($src3$$reg));
11039   %}
11040 
11041   ins_pipe(imac_reg_reg);
11042 %}
11043 
11044 // Combined Integer Multiply & Neg
11045 
11046 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11047   match(Set dst (MulI (SubI zero src1) src2));
11048   match(Set dst (MulI src1 (SubI zero src2)));
11049 
11050   ins_cost(INSN_COST * 3);
11051   format %{ "mneg  $dst, $src1, $src2" %}
11052 
11053   ins_encode %{
11054     __ mnegw(as_Register($dst$$reg),
11055              as_Register($src1$$reg),
11056              as_Register($src2$$reg));
11057   %}
11058 
11059   ins_pipe(imac_reg_reg);
11060 %}
11061 
11062 // Combined Long Multiply & Add/Sub
11063 
11064 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11065   match(Set dst (AddL src3 (MulL src1 src2)));
11066 
11067   ins_cost(INSN_COST * 5);
11068   format %{ "madd  $dst, $src1, $src2, $src3" %}
11069 
11070   ins_encode %{
11071     __ madd(as_Register($dst$$reg),
11072             as_Register($src1$$reg),
11073             as_Register($src2$$reg),
11074             as_Register($src3$$reg));
11075   %}
11076 
11077   ins_pipe(lmac_reg_reg);
11078 %}
11079 
11080 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11081   match(Set dst (SubL src3 (MulL src1 src2)));
11082 
11083   ins_cost(INSN_COST * 5);
11084   format %{ "msub  $dst, $src1, $src2, $src3" %}
11085 
11086   ins_encode %{
11087     __ msub(as_Register($dst$$reg),
11088             as_Register($src1$$reg),
11089             as_Register($src2$$reg),
11090             as_Register($src3$$reg));
11091   %}
11092 
11093   ins_pipe(lmac_reg_reg);
11094 %}
11095 
11096 // Combined Long Multiply & Neg
11097 
11098 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11099   match(Set dst (MulL (SubL zero src1) src2));
11100   match(Set dst (MulL src1 (SubL zero src2)));
11101 
11102   ins_cost(INSN_COST * 5);
11103   format %{ "mneg  $dst, $src1, $src2" %}
11104 
11105   ins_encode %{
11106     __ mneg(as_Register($dst$$reg),
11107             as_Register($src1$$reg),
11108             as_Register($src2$$reg));
11109   %}
11110 
11111   ins_pipe(lmac_reg_reg);
11112 %}
11113 
11114 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11115 
11116 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11117   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11118 
11119   ins_cost(INSN_COST * 3);
11120   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11121 
11122   ins_encode %{
11123     __ smaddl(as_Register($dst$$reg),
11124               as_Register($src1$$reg),
11125               as_Register($src2$$reg),
11126               as_Register($src3$$reg));
11127   %}
11128 
11129   ins_pipe(imac_reg_reg);
11130 %}
11131 
11132 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11133   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11134 
11135   ins_cost(INSN_COST * 3);
11136   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11137 
11138   ins_encode %{
11139     __ smsubl(as_Register($dst$$reg),
11140               as_Register($src1$$reg),
11141               as_Register($src2$$reg),
11142               as_Register($src3$$reg));
11143   %}
11144 
11145   ins_pipe(imac_reg_reg);
11146 %}
11147 
11148 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11149   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11150   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11151 
11152   ins_cost(INSN_COST * 3);
11153   format %{ "smnegl  $dst, $src1, $src2" %}
11154 
11155   ins_encode %{
11156     __ smnegl(as_Register($dst$$reg),
11157               as_Register($src1$$reg),
11158               as_Register($src2$$reg));
11159   %}
11160 
11161   ins_pipe(imac_reg_reg);
11162 %}
11163 
11164 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11165 
11166 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11167   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11168 
11169   ins_cost(INSN_COST * 5);
11170   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11171             "maddw $dst, $src3, $src4, rscratch1" %}
11172 
11173   ins_encode %{
11174     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11175     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11176 
11177   ins_pipe(imac_reg_reg);
11178 %}
11179 
11180 // Integer Divide
11181 
11182 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11183   match(Set dst (DivI src1 src2));
11184 
11185   ins_cost(INSN_COST * 19);
11186   format %{ "sdivw  $dst, $src1, $src2" %}
11187 
11188   ins_encode(aarch64_enc_divw(dst, src1, src2));
11189   ins_pipe(idiv_reg_reg);
11190 %}
11191 
11192 // Long Divide
11193 
11194 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11195   match(Set dst (DivL src1 src2));
11196 
11197   ins_cost(INSN_COST * 35);
11198   format %{ "sdiv   $dst, $src1, $src2" %}
11199 
11200   ins_encode(aarch64_enc_div(dst, src1, src2));
11201   ins_pipe(ldiv_reg_reg);
11202 %}
11203 
11204 // Integer Remainder
11205 
11206 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11207   match(Set dst (ModI src1 src2));
11208 
11209   ins_cost(INSN_COST * 22);
11210   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11211             "msubw($dst, rscratch1, $src2, $src1" %}
11212 
11213   ins_encode(aarch64_enc_modw(dst, src1, src2));
11214   ins_pipe(idiv_reg_reg);
11215 %}
11216 
11217 // Long Remainder
11218 
11219 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11220   match(Set dst (ModL src1 src2));
11221 
11222   ins_cost(INSN_COST * 38);
11223   format %{ "sdiv   rscratch1, $src1, $src2\n"
11224             "msub($dst, rscratch1, $src2, $src1" %}
11225 
11226   ins_encode(aarch64_enc_mod(dst, src1, src2));
11227   ins_pipe(ldiv_reg_reg);
11228 %}
11229 
11230 // Integer Shifts
11231 
11232 // Shift Left Register
11233 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11234   match(Set dst (LShiftI src1 src2));
11235 
11236   ins_cost(INSN_COST * 2);
11237   format %{ "lslvw  $dst, $src1, $src2" %}
11238 
11239   ins_encode %{
11240     __ lslvw(as_Register($dst$$reg),
11241              as_Register($src1$$reg),
11242              as_Register($src2$$reg));
11243   %}
11244 
11245   ins_pipe(ialu_reg_reg_vshift);
11246 %}
11247 
11248 // Shift Left Immediate
11249 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11250   match(Set dst (LShiftI src1 src2));
11251 
11252   ins_cost(INSN_COST);
11253   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11254 
11255   ins_encode %{
11256     __ lslw(as_Register($dst$$reg),
11257             as_Register($src1$$reg),
11258             $src2$$constant & 0x1f);
11259   %}
11260 
11261   ins_pipe(ialu_reg_shift);
11262 %}
11263 
11264 // Shift Right Logical Register
11265 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11266   match(Set dst (URShiftI src1 src2));
11267 
11268   ins_cost(INSN_COST * 2);
11269   format %{ "lsrvw  $dst, $src1, $src2" %}
11270 
11271   ins_encode %{
11272     __ lsrvw(as_Register($dst$$reg),
11273              as_Register($src1$$reg),
11274              as_Register($src2$$reg));
11275   %}
11276 
11277   ins_pipe(ialu_reg_reg_vshift);
11278 %}
11279 
11280 // Shift Right Logical Immediate
11281 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11282   match(Set dst (URShiftI src1 src2));
11283 
11284   ins_cost(INSN_COST);
11285   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11286 
11287   ins_encode %{
11288     __ lsrw(as_Register($dst$$reg),
11289             as_Register($src1$$reg),
11290             $src2$$constant & 0x1f);
11291   %}
11292 
11293   ins_pipe(ialu_reg_shift);
11294 %}
11295 
11296 // Shift Right Arithmetic Register
11297 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11298   match(Set dst (RShiftI src1 src2));
11299 
11300   ins_cost(INSN_COST * 2);
11301   format %{ "asrvw  $dst, $src1, $src2" %}
11302 
11303   ins_encode %{
11304     __ asrvw(as_Register($dst$$reg),
11305              as_Register($src1$$reg),
11306              as_Register($src2$$reg));
11307   %}
11308 
11309   ins_pipe(ialu_reg_reg_vshift);
11310 %}
11311 
11312 // Shift Right Arithmetic Immediate
11313 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11314   match(Set dst (RShiftI src1 src2));
11315 
11316   ins_cost(INSN_COST);
11317   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11318 
11319   ins_encode %{
11320     __ asrw(as_Register($dst$$reg),
11321             as_Register($src1$$reg),
11322             $src2$$constant & 0x1f);
11323   %}
11324 
11325   ins_pipe(ialu_reg_shift);
11326 %}
11327 
11328 // Combined Int Mask and Right Shift (using UBFM)
11329 // TODO
11330 
11331 // Long Shifts
11332 
11333 // Shift Left Register
11334 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11335   match(Set dst (LShiftL src1 src2));
11336 
11337   ins_cost(INSN_COST * 2);
11338   format %{ "lslv  $dst, $src1, $src2" %}
11339 
11340   ins_encode %{
11341     __ lslv(as_Register($dst$$reg),
11342             as_Register($src1$$reg),
11343             as_Register($src2$$reg));
11344   %}
11345 
11346   ins_pipe(ialu_reg_reg_vshift);
11347 %}
11348 
11349 // Shift Left Immediate
11350 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11351   match(Set dst (LShiftL src1 src2));
11352 
11353   ins_cost(INSN_COST);
11354   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11355 
11356   ins_encode %{
11357     __ lsl(as_Register($dst$$reg),
11358             as_Register($src1$$reg),
11359             $src2$$constant & 0x3f);
11360   %}
11361 
11362   ins_pipe(ialu_reg_shift);
11363 %}
11364 
11365 // Shift Right Logical Register
11366 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11367   match(Set dst (URShiftL src1 src2));
11368 
11369   ins_cost(INSN_COST * 2);
11370   format %{ "lsrv  $dst, $src1, $src2" %}
11371 
11372   ins_encode %{
11373     __ lsrv(as_Register($dst$$reg),
11374             as_Register($src1$$reg),
11375             as_Register($src2$$reg));
11376   %}
11377 
11378   ins_pipe(ialu_reg_reg_vshift);
11379 %}
11380 
11381 // Shift Right Logical Immediate
11382 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11383   match(Set dst (URShiftL src1 src2));
11384 
11385   ins_cost(INSN_COST);
11386   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11387 
11388   ins_encode %{
11389     __ lsr(as_Register($dst$$reg),
11390            as_Register($src1$$reg),
11391            $src2$$constant & 0x3f);
11392   %}
11393 
11394   ins_pipe(ialu_reg_shift);
11395 %}
11396 
11397 // A special-case pattern for card table stores.
11398 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11399   match(Set dst (URShiftL (CastP2X src1) src2));
11400 
11401   ins_cost(INSN_COST);
11402   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11403 
11404   ins_encode %{
11405     __ lsr(as_Register($dst$$reg),
11406            as_Register($src1$$reg),
11407            $src2$$constant & 0x3f);
11408   %}
11409 
11410   ins_pipe(ialu_reg_shift);
11411 %}
11412 
11413 // Shift Right Arithmetic Register
11414 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11415   match(Set dst (RShiftL src1 src2));
11416 
11417   ins_cost(INSN_COST * 2);
11418   format %{ "asrv  $dst, $src1, $src2" %}
11419 
11420   ins_encode %{
11421     __ asrv(as_Register($dst$$reg),
11422             as_Register($src1$$reg),
11423             as_Register($src2$$reg));
11424   %}
11425 
11426   ins_pipe(ialu_reg_reg_vshift);
11427 %}
11428 
11429 // Shift Right Arithmetic Immediate
11430 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11431   match(Set dst (RShiftL src1 src2));
11432 
11433   ins_cost(INSN_COST);
11434   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11435 
11436   ins_encode %{
11437     __ asr(as_Register($dst$$reg),
11438            as_Register($src1$$reg),
11439            $src2$$constant & 0x3f);
11440   %}
11441 
11442   ins_pipe(ialu_reg_shift);
11443 %}
11444 
11445 // BEGIN This section of the file is automatically generated. Do not edit --------------
11446 // This section is generated from aarch64_ad.m4
11447 
11448 
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 instruct regL_not_reg(iRegLNoSp dst,
11452                          iRegL src1, immL_M1 m1,
11453                          rFlagsReg cr) %{
11454   match(Set dst (XorL src1 m1));
11455   ins_cost(INSN_COST);
11456   format %{ "eon  $dst, $src1, zr" %}
11457 
11458   ins_encode %{
11459     __ eon(as_Register($dst$$reg),
11460               as_Register($src1$$reg),
11461               zr,
11462               Assembler::LSL, 0);
11463   %}
11464 
11465   ins_pipe(ialu_reg);
11466 %}
11467 
11468 // This pattern is automatically generated from aarch64_ad.m4
11469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11470 instruct regI_not_reg(iRegINoSp dst,
11471                          iRegIorL2I src1, immI_M1 m1,
11472                          rFlagsReg cr) %{
11473   match(Set dst (XorI src1 m1));
11474   ins_cost(INSN_COST);
11475   format %{ "eonw  $dst, $src1, zr" %}
11476 
11477   ins_encode %{
11478     __ eonw(as_Register($dst$$reg),
11479               as_Register($src1$$reg),
11480               zr,
11481               Assembler::LSL, 0);
11482   %}
11483 
11484   ins_pipe(ialu_reg);
11485 %}
11486 
11487 // This pattern is automatically generated from aarch64_ad.m4
11488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11489 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11490                               immI0 zero, iRegIorL2I src1, immI src2) %{
11491   match(Set dst (SubI zero (URShiftI src1 src2)));
11492 
11493   ins_cost(1.9 * INSN_COST);
11494   format %{ "negw  $dst, $src1, LSR $src2" %}
11495 
11496   ins_encode %{
11497     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11498             Assembler::LSR, $src2$$constant & 0x1f);
11499   %}
11500 
11501   ins_pipe(ialu_reg_shift);
11502 %}
11503 
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11507                               immI0 zero, iRegIorL2I src1, immI src2) %{
11508   match(Set dst (SubI zero (RShiftI src1 src2)));
11509 
11510   ins_cost(1.9 * INSN_COST);
11511   format %{ "negw  $dst, $src1, ASR $src2" %}
11512 
11513   ins_encode %{
11514     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11515             Assembler::ASR, $src2$$constant & 0x1f);
11516   %}
11517 
11518   ins_pipe(ialu_reg_shift);
11519 %}
11520 
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11524                               immI0 zero, iRegIorL2I src1, immI src2) %{
11525   match(Set dst (SubI zero (LShiftI src1 src2)));
11526 
11527   ins_cost(1.9 * INSN_COST);
11528   format %{ "negw  $dst, $src1, LSL $src2" %}
11529 
11530   ins_encode %{
11531     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11532             Assembler::LSL, $src2$$constant & 0x1f);
11533   %}
11534 
11535   ins_pipe(ialu_reg_shift);
11536 %}
11537 
11538 // This pattern is automatically generated from aarch64_ad.m4
11539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11540 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11541                               immL0 zero, iRegL src1, immI src2) %{
11542   match(Set dst (SubL zero (URShiftL src1 src2)));
11543 
11544   ins_cost(1.9 * INSN_COST);
11545   format %{ "neg  $dst, $src1, LSR $src2" %}
11546 
11547   ins_encode %{
11548     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11549             Assembler::LSR, $src2$$constant & 0x3f);
11550   %}
11551 
11552   ins_pipe(ialu_reg_shift);
11553 %}
11554 
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11558                               immL0 zero, iRegL src1, immI src2) %{
11559   match(Set dst (SubL zero (RShiftL src1 src2)));
11560 
11561   ins_cost(1.9 * INSN_COST);
11562   format %{ "neg  $dst, $src1, ASR $src2" %}
11563 
11564   ins_encode %{
11565     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11566             Assembler::ASR, $src2$$constant & 0x3f);
11567   %}
11568 
11569   ins_pipe(ialu_reg_shift);
11570 %}
11571 
11572 // This pattern is automatically generated from aarch64_ad.m4
11573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11574 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11575                               immL0 zero, iRegL src1, immI src2) %{
11576   match(Set dst (SubL zero (LShiftL src1 src2)));
11577 
11578   ins_cost(1.9 * INSN_COST);
11579   format %{ "neg  $dst, $src1, LSL $src2" %}
11580 
11581   ins_encode %{
11582     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11583             Assembler::LSL, $src2$$constant & 0x3f);
11584   %}
11585 
11586   ins_pipe(ialu_reg_shift);
11587 %}
11588 
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591 instruct AndI_reg_not_reg(iRegINoSp dst,
11592                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11593   match(Set dst (AndI src1 (XorI src2 m1)));
11594   ins_cost(INSN_COST);
11595   format %{ "bicw  $dst, $src1, $src2" %}
11596 
11597   ins_encode %{
11598     __ bicw(as_Register($dst$$reg),
11599               as_Register($src1$$reg),
11600               as_Register($src2$$reg),
11601               Assembler::LSL, 0);
11602   %}
11603 
11604   ins_pipe(ialu_reg_reg);
11605 %}
11606 
11607 // This pattern is automatically generated from aarch64_ad.m4
11608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11609 instruct AndL_reg_not_reg(iRegLNoSp dst,
11610                          iRegL src1, iRegL src2, immL_M1 m1) %{
11611   match(Set dst (AndL src1 (XorL src2 m1)));
11612   ins_cost(INSN_COST);
11613   format %{ "bic  $dst, $src1, $src2" %}
11614 
11615   ins_encode %{
11616     __ bic(as_Register($dst$$reg),
11617               as_Register($src1$$reg),
11618               as_Register($src2$$reg),
11619               Assembler::LSL, 0);
11620   %}
11621 
11622   ins_pipe(ialu_reg_reg);
11623 %}
11624 
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 instruct OrI_reg_not_reg(iRegINoSp dst,
11628                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11629   match(Set dst (OrI src1 (XorI src2 m1)));
11630   ins_cost(INSN_COST);
11631   format %{ "ornw  $dst, $src1, $src2" %}
11632 
11633   ins_encode %{
11634     __ ornw(as_Register($dst$$reg),
11635               as_Register($src1$$reg),
11636               as_Register($src2$$reg),
11637               Assembler::LSL, 0);
11638   %}
11639 
11640   ins_pipe(ialu_reg_reg);
11641 %}
11642 
11643 // This pattern is automatically generated from aarch64_ad.m4
11644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11645 instruct OrL_reg_not_reg(iRegLNoSp dst,
11646                          iRegL src1, iRegL src2, immL_M1 m1) %{
11647   match(Set dst (OrL src1 (XorL src2 m1)));
11648   ins_cost(INSN_COST);
11649   format %{ "orn  $dst, $src1, $src2" %}
11650 
11651   ins_encode %{
11652     __ orn(as_Register($dst$$reg),
11653               as_Register($src1$$reg),
11654               as_Register($src2$$reg),
11655               Assembler::LSL, 0);
11656   %}
11657 
11658   ins_pipe(ialu_reg_reg);
11659 %}
11660 
11661 // This pattern is automatically generated from aarch64_ad.m4
11662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11663 instruct XorI_reg_not_reg(iRegINoSp dst,
11664                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11665   match(Set dst (XorI m1 (XorI src2 src1)));
11666   ins_cost(INSN_COST);
11667   format %{ "eonw  $dst, $src1, $src2" %}
11668 
11669   ins_encode %{
11670     __ eonw(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::LSL, 0);
11674   %}
11675 
11676   ins_pipe(ialu_reg_reg);
11677 %}
11678 
11679 // This pattern is automatically generated from aarch64_ad.m4
11680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11681 instruct XorL_reg_not_reg(iRegLNoSp dst,
11682                          iRegL src1, iRegL src2, immL_M1 m1) %{
11683   match(Set dst (XorL m1 (XorL src2 src1)));
11684   ins_cost(INSN_COST);
11685   format %{ "eon  $dst, $src1, $src2" %}
11686 
11687   ins_encode %{
11688     __ eon(as_Register($dst$$reg),
11689               as_Register($src1$$reg),
11690               as_Register($src2$$reg),
11691               Assembler::LSL, 0);
11692   %}
11693 
11694   ins_pipe(ialu_reg_reg);
11695 %}
11696 
11697 // This pattern is automatically generated from aarch64_ad.m4
11698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11699 // val & (-1 ^ (val >>> shift)) ==> bicw
11700 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11701                          iRegIorL2I src1, iRegIorL2I src2,
11702                          immI src3, immI_M1 src4) %{
11703   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11704   ins_cost(1.9 * INSN_COST);
11705   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11706 
11707   ins_encode %{
11708     __ bicw(as_Register($dst$$reg),
11709               as_Register($src1$$reg),
11710               as_Register($src2$$reg),
11711               Assembler::LSR,
11712               $src3$$constant & 0x1f);
11713   %}
11714 
11715   ins_pipe(ialu_reg_reg_shift);
11716 %}
11717 
11718 // This pattern is automatically generated from aarch64_ad.m4
11719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11720 // val & (-1 ^ (val >>> shift)) ==> bic
11721 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11722                          iRegL src1, iRegL src2,
11723                          immI src3, immL_M1 src4) %{
11724   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11725   ins_cost(1.9 * INSN_COST);
11726   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11727 
11728   ins_encode %{
11729     __ bic(as_Register($dst$$reg),
11730               as_Register($src1$$reg),
11731               as_Register($src2$$reg),
11732               Assembler::LSR,
11733               $src3$$constant & 0x3f);
11734   %}
11735 
11736   ins_pipe(ialu_reg_reg_shift);
11737 %}
11738 
11739 // This pattern is automatically generated from aarch64_ad.m4
11740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11741 // val & (-1 ^ (val >> shift)) ==> bicw
11742 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11743                          iRegIorL2I src1, iRegIorL2I src2,
11744                          immI src3, immI_M1 src4) %{
11745   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11746   ins_cost(1.9 * INSN_COST);
11747   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11748 
11749   ins_encode %{
11750     __ bicw(as_Register($dst$$reg),
11751               as_Register($src1$$reg),
11752               as_Register($src2$$reg),
11753               Assembler::ASR,
11754               $src3$$constant & 0x1f);
11755   %}
11756 
11757   ins_pipe(ialu_reg_reg_shift);
11758 %}
11759 
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 // val & (-1 ^ (val >> shift)) ==> bic
11763 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11764                          iRegL src1, iRegL src2,
11765                          immI src3, immL_M1 src4) %{
11766   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11767   ins_cost(1.9 * INSN_COST);
11768   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11769 
11770   ins_encode %{
11771     __ bic(as_Register($dst$$reg),
11772               as_Register($src1$$reg),
11773               as_Register($src2$$reg),
11774               Assembler::ASR,
11775               $src3$$constant & 0x3f);
11776   %}
11777 
11778   ins_pipe(ialu_reg_reg_shift);
11779 %}
11780 
11781 // This pattern is automatically generated from aarch64_ad.m4
11782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11783 // val & (-1 ^ (val ror shift)) ==> bicw
11784 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11785                          iRegIorL2I src1, iRegIorL2I src2,
11786                          immI src3, immI_M1 src4) %{
11787   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11788   ins_cost(1.9 * INSN_COST);
11789   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11790 
11791   ins_encode %{
11792     __ bicw(as_Register($dst$$reg),
11793               as_Register($src1$$reg),
11794               as_Register($src2$$reg),
11795               Assembler::ROR,
11796               $src3$$constant & 0x1f);
11797   %}
11798 
11799   ins_pipe(ialu_reg_reg_shift);
11800 %}
11801 
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 // val & (-1 ^ (val ror shift)) ==> bic
11805 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11806                          iRegL src1, iRegL src2,
11807                          immI src3, immL_M1 src4) %{
11808   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11809   ins_cost(1.9 * INSN_COST);
11810   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11811 
11812   ins_encode %{
11813     __ bic(as_Register($dst$$reg),
11814               as_Register($src1$$reg),
11815               as_Register($src2$$reg),
11816               Assembler::ROR,
11817               $src3$$constant & 0x3f);
11818   %}
11819 
11820   ins_pipe(ialu_reg_reg_shift);
11821 %}
11822 
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 // val & (-1 ^ (val << shift)) ==> bicw
11826 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11827                          iRegIorL2I src1, iRegIorL2I src2,
11828                          immI src3, immI_M1 src4) %{
11829   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11830   ins_cost(1.9 * INSN_COST);
11831   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11832 
11833   ins_encode %{
11834     __ bicw(as_Register($dst$$reg),
11835               as_Register($src1$$reg),
11836               as_Register($src2$$reg),
11837               Assembler::LSL,
11838               $src3$$constant & 0x1f);
11839   %}
11840 
11841   ins_pipe(ialu_reg_reg_shift);
11842 %}
11843 
11844 // This pattern is automatically generated from aarch64_ad.m4
11845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11846 // val & (-1 ^ (val << shift)) ==> bic
11847 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11848                          iRegL src1, iRegL src2,
11849                          immI src3, immL_M1 src4) %{
11850   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11851   ins_cost(1.9 * INSN_COST);
11852   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11853 
11854   ins_encode %{
11855     __ bic(as_Register($dst$$reg),
11856               as_Register($src1$$reg),
11857               as_Register($src2$$reg),
11858               Assembler::LSL,
11859               $src3$$constant & 0x3f);
11860   %}
11861 
11862   ins_pipe(ialu_reg_reg_shift);
11863 %}
11864 
11865 // This pattern is automatically generated from aarch64_ad.m4
11866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11867 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11868 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11869                          iRegIorL2I src1, iRegIorL2I src2,
11870                          immI src3, immI_M1 src4) %{
11871   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11872   ins_cost(1.9 * INSN_COST);
11873   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11874 
11875   ins_encode %{
11876     __ eonw(as_Register($dst$$reg),
11877               as_Register($src1$$reg),
11878               as_Register($src2$$reg),
11879               Assembler::LSR,
11880               $src3$$constant & 0x1f);
11881   %}
11882 
11883   ins_pipe(ialu_reg_reg_shift);
11884 %}
11885 
11886 // This pattern is automatically generated from aarch64_ad.m4
11887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11888 // val ^ (-1 ^ (val >>> shift)) ==> eon
11889 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11890                          iRegL src1, iRegL src2,
11891                          immI src3, immL_M1 src4) %{
11892   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11893   ins_cost(1.9 * INSN_COST);
11894   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11895 
11896   ins_encode %{
11897     __ eon(as_Register($dst$$reg),
11898               as_Register($src1$$reg),
11899               as_Register($src2$$reg),
11900               Assembler::LSR,
11901               $src3$$constant & 0x3f);
11902   %}
11903 
11904   ins_pipe(ialu_reg_reg_shift);
11905 %}
11906 
11907 // This pattern is automatically generated from aarch64_ad.m4
11908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11909 // val ^ (-1 ^ (val >> shift)) ==> eonw
11910 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11911                          iRegIorL2I src1, iRegIorL2I src2,
11912                          immI src3, immI_M1 src4) %{
11913   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11914   ins_cost(1.9 * INSN_COST);
11915   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11916 
11917   ins_encode %{
11918     __ eonw(as_Register($dst$$reg),
11919               as_Register($src1$$reg),
11920               as_Register($src2$$reg),
11921               Assembler::ASR,
11922               $src3$$constant & 0x1f);
11923   %}
11924 
11925   ins_pipe(ialu_reg_reg_shift);
11926 %}
11927 
11928 // This pattern is automatically generated from aarch64_ad.m4
11929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11930 // val ^ (-1 ^ (val >> shift)) ==> eon
11931 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11932                          iRegL src1, iRegL src2,
11933                          immI src3, immL_M1 src4) %{
11934   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11935   ins_cost(1.9 * INSN_COST);
11936   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11937 
11938   ins_encode %{
11939     __ eon(as_Register($dst$$reg),
11940               as_Register($src1$$reg),
11941               as_Register($src2$$reg),
11942               Assembler::ASR,
11943               $src3$$constant & 0x3f);
11944   %}
11945 
11946   ins_pipe(ialu_reg_reg_shift);
11947 %}
11948 
11949 // This pattern is automatically generated from aarch64_ad.m4
11950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11951 // val ^ (-1 ^ (val ror shift)) ==> eonw
11952 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11953                          iRegIorL2I src1, iRegIorL2I src2,
11954                          immI src3, immI_M1 src4) %{
11955   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11956   ins_cost(1.9 * INSN_COST);
11957   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11958 
11959   ins_encode %{
11960     __ eonw(as_Register($dst$$reg),
11961               as_Register($src1$$reg),
11962               as_Register($src2$$reg),
11963               Assembler::ROR,
11964               $src3$$constant & 0x1f);
11965   %}
11966 
11967   ins_pipe(ialu_reg_reg_shift);
11968 %}
11969 
11970 // This pattern is automatically generated from aarch64_ad.m4
11971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11972 // val ^ (-1 ^ (val ror shift)) ==> eon
11973 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11974                          iRegL src1, iRegL src2,
11975                          immI src3, immL_M1 src4) %{
11976   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11977   ins_cost(1.9 * INSN_COST);
11978   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11979 
11980   ins_encode %{
11981     __ eon(as_Register($dst$$reg),
11982               as_Register($src1$$reg),
11983               as_Register($src2$$reg),
11984               Assembler::ROR,
11985               $src3$$constant & 0x3f);
11986   %}
11987 
11988   ins_pipe(ialu_reg_reg_shift);
11989 %}
11990 
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 // val ^ (-1 ^ (val << shift)) ==> eonw
11994 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11995                          iRegIorL2I src1, iRegIorL2I src2,
11996                          immI src3, immI_M1 src4) %{
11997   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11998   ins_cost(1.9 * INSN_COST);
11999   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
12000 
12001   ins_encode %{
12002     __ eonw(as_Register($dst$$reg),
12003               as_Register($src1$$reg),
12004               as_Register($src2$$reg),
12005               Assembler::LSL,
12006               $src3$$constant & 0x1f);
12007   %}
12008 
12009   ins_pipe(ialu_reg_reg_shift);
12010 %}
12011 
12012 // This pattern is automatically generated from aarch64_ad.m4
12013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12014 // val ^ (-1 ^ (val << shift)) ==> eon
12015 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12016                          iRegL src1, iRegL src2,
12017                          immI src3, immL_M1 src4) %{
12018   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12019   ins_cost(1.9 * INSN_COST);
12020   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12021 
12022   ins_encode %{
12023     __ eon(as_Register($dst$$reg),
12024               as_Register($src1$$reg),
12025               as_Register($src2$$reg),
12026               Assembler::LSL,
12027               $src3$$constant & 0x3f);
12028   %}
12029 
12030   ins_pipe(ialu_reg_reg_shift);
12031 %}
12032 
12033 // This pattern is automatically generated from aarch64_ad.m4
12034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12035 // val | (-1 ^ (val >>> shift)) ==> ornw
12036 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12037                          iRegIorL2I src1, iRegIorL2I src2,
12038                          immI src3, immI_M1 src4) %{
12039   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12040   ins_cost(1.9 * INSN_COST);
12041   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12042 
12043   ins_encode %{
12044     __ ornw(as_Register($dst$$reg),
12045               as_Register($src1$$reg),
12046               as_Register($src2$$reg),
12047               Assembler::LSR,
12048               $src3$$constant & 0x1f);
12049   %}
12050 
12051   ins_pipe(ialu_reg_reg_shift);
12052 %}
12053 
12054 // This pattern is automatically generated from aarch64_ad.m4
12055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12056 // val | (-1 ^ (val >>> shift)) ==> orn
12057 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12058                          iRegL src1, iRegL src2,
12059                          immI src3, immL_M1 src4) %{
12060   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12061   ins_cost(1.9 * INSN_COST);
12062   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12063 
12064   ins_encode %{
12065     __ orn(as_Register($dst$$reg),
12066               as_Register($src1$$reg),
12067               as_Register($src2$$reg),
12068               Assembler::LSR,
12069               $src3$$constant & 0x3f);
12070   %}
12071 
12072   ins_pipe(ialu_reg_reg_shift);
12073 %}
12074 
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 // val | (-1 ^ (val >> shift)) ==> ornw
12078 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12079                          iRegIorL2I src1, iRegIorL2I src2,
12080                          immI src3, immI_M1 src4) %{
12081   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12082   ins_cost(1.9 * INSN_COST);
12083   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12084 
12085   ins_encode %{
12086     __ ornw(as_Register($dst$$reg),
12087               as_Register($src1$$reg),
12088               as_Register($src2$$reg),
12089               Assembler::ASR,
12090               $src3$$constant & 0x1f);
12091   %}
12092 
12093   ins_pipe(ialu_reg_reg_shift);
12094 %}
12095 
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 // val | (-1 ^ (val >> shift)) ==> orn
12099 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12100                          iRegL src1, iRegL src2,
12101                          immI src3, immL_M1 src4) %{
12102   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12103   ins_cost(1.9 * INSN_COST);
12104   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12105 
12106   ins_encode %{
12107     __ orn(as_Register($dst$$reg),
12108               as_Register($src1$$reg),
12109               as_Register($src2$$reg),
12110               Assembler::ASR,
12111               $src3$$constant & 0x3f);
12112   %}
12113 
12114   ins_pipe(ialu_reg_reg_shift);
12115 %}
12116 
12117 // This pattern is automatically generated from aarch64_ad.m4
12118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12119 // val | (-1 ^ (val ror shift)) ==> ornw
12120 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12121                          iRegIorL2I src1, iRegIorL2I src2,
12122                          immI src3, immI_M1 src4) %{
12123   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12124   ins_cost(1.9 * INSN_COST);
12125   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12126 
12127   ins_encode %{
12128     __ ornw(as_Register($dst$$reg),
12129               as_Register($src1$$reg),
12130               as_Register($src2$$reg),
12131               Assembler::ROR,
12132               $src3$$constant & 0x1f);
12133   %}
12134 
12135   ins_pipe(ialu_reg_reg_shift);
12136 %}
12137 
12138 // This pattern is automatically generated from aarch64_ad.m4
12139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12140 // val | (-1 ^ (val ror shift)) ==> orn
12141 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12142                          iRegL src1, iRegL src2,
12143                          immI src3, immL_M1 src4) %{
12144   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12145   ins_cost(1.9 * INSN_COST);
12146   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12147 
12148   ins_encode %{
12149     __ orn(as_Register($dst$$reg),
12150               as_Register($src1$$reg),
12151               as_Register($src2$$reg),
12152               Assembler::ROR,
12153               $src3$$constant & 0x3f);
12154   %}
12155 
12156   ins_pipe(ialu_reg_reg_shift);
12157 %}
12158 
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161 // val | (-1 ^ (val << shift)) ==> ornw
12162 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12163                          iRegIorL2I src1, iRegIorL2I src2,
12164                          immI src3, immI_M1 src4) %{
12165   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12166   ins_cost(1.9 * INSN_COST);
12167   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12168 
12169   ins_encode %{
12170     __ ornw(as_Register($dst$$reg),
12171               as_Register($src1$$reg),
12172               as_Register($src2$$reg),
12173               Assembler::LSL,
12174               $src3$$constant & 0x1f);
12175   %}
12176 
12177   ins_pipe(ialu_reg_reg_shift);
12178 %}
12179 
12180 // This pattern is automatically generated from aarch64_ad.m4
12181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12182 // val | (-1 ^ (val << shift)) ==> orn
12183 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12184                          iRegL src1, iRegL src2,
12185                          immI src3, immL_M1 src4) %{
12186   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12187   ins_cost(1.9 * INSN_COST);
12188   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12189 
12190   ins_encode %{
12191     __ orn(as_Register($dst$$reg),
12192               as_Register($src1$$reg),
12193               as_Register($src2$$reg),
12194               Assembler::LSL,
12195               $src3$$constant & 0x3f);
12196   %}
12197 
12198   ins_pipe(ialu_reg_reg_shift);
12199 %}
12200 
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12204                          iRegIorL2I src1, iRegIorL2I src2,
12205                          immI src3) %{
12206   match(Set dst (AndI src1 (URShiftI src2 src3)));
12207 
12208   ins_cost(1.9 * INSN_COST);
12209   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12210 
12211   ins_encode %{
12212     __ andw(as_Register($dst$$reg),
12213               as_Register($src1$$reg),
12214               as_Register($src2$$reg),
12215               Assembler::LSR,
12216               $src3$$constant & 0x1f);
12217   %}
12218 
12219   ins_pipe(ialu_reg_reg_shift);
12220 %}
12221 
12222 // This pattern is automatically generated from aarch64_ad.m4
12223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12224 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12225                          iRegL src1, iRegL src2,
12226                          immI src3) %{
12227   match(Set dst (AndL src1 (URShiftL src2 src3)));
12228 
12229   ins_cost(1.9 * INSN_COST);
12230   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12231 
12232   ins_encode %{
12233     __ andr(as_Register($dst$$reg),
12234               as_Register($src1$$reg),
12235               as_Register($src2$$reg),
12236               Assembler::LSR,
12237               $src3$$constant & 0x3f);
12238   %}
12239 
12240   ins_pipe(ialu_reg_reg_shift);
12241 %}
12242 
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12246                          iRegIorL2I src1, iRegIorL2I src2,
12247                          immI src3) %{
12248   match(Set dst (AndI src1 (RShiftI src2 src3)));
12249 
12250   ins_cost(1.9 * INSN_COST);
12251   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12252 
12253   ins_encode %{
12254     __ andw(as_Register($dst$$reg),
12255               as_Register($src1$$reg),
12256               as_Register($src2$$reg),
12257               Assembler::ASR,
12258               $src3$$constant & 0x1f);
12259   %}
12260 
12261   ins_pipe(ialu_reg_reg_shift);
12262 %}
12263 
12264 // This pattern is automatically generated from aarch64_ad.m4
12265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12266 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12267                          iRegL src1, iRegL src2,
12268                          immI src3) %{
12269   match(Set dst (AndL src1 (RShiftL src2 src3)));
12270 
12271   ins_cost(1.9 * INSN_COST);
12272   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12273 
12274   ins_encode %{
12275     __ andr(as_Register($dst$$reg),
12276               as_Register($src1$$reg),
12277               as_Register($src2$$reg),
12278               Assembler::ASR,
12279               $src3$$constant & 0x3f);
12280   %}
12281 
12282   ins_pipe(ialu_reg_reg_shift);
12283 %}
12284 
12285 // This pattern is automatically generated from aarch64_ad.m4
12286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12287 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12288                          iRegIorL2I src1, iRegIorL2I src2,
12289                          immI src3) %{
12290   match(Set dst (AndI src1 (LShiftI src2 src3)));
12291 
12292   ins_cost(1.9 * INSN_COST);
12293   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12294 
12295   ins_encode %{
12296     __ andw(as_Register($dst$$reg),
12297               as_Register($src1$$reg),
12298               as_Register($src2$$reg),
12299               Assembler::LSL,
12300               $src3$$constant & 0x1f);
12301   %}
12302 
12303   ins_pipe(ialu_reg_reg_shift);
12304 %}
12305 
12306 // This pattern is automatically generated from aarch64_ad.m4
12307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12308 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12309                          iRegL src1, iRegL src2,
12310                          immI src3) %{
12311   match(Set dst (AndL src1 (LShiftL src2 src3)));
12312 
12313   ins_cost(1.9 * INSN_COST);
12314   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12315 
12316   ins_encode %{
12317     __ andr(as_Register($dst$$reg),
12318               as_Register($src1$$reg),
12319               as_Register($src2$$reg),
12320               Assembler::LSL,
12321               $src3$$constant & 0x3f);
12322   %}
12323 
12324   ins_pipe(ialu_reg_reg_shift);
12325 %}
12326 
12327 // This pattern is automatically generated from aarch64_ad.m4
12328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12329 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12330                          iRegIorL2I src1, iRegIorL2I src2,
12331                          immI src3) %{
12332   match(Set dst (AndI src1 (RotateRight src2 src3)));
12333 
12334   ins_cost(1.9 * INSN_COST);
12335   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12336 
12337   ins_encode %{
12338     __ andw(as_Register($dst$$reg),
12339               as_Register($src1$$reg),
12340               as_Register($src2$$reg),
12341               Assembler::ROR,
12342               $src3$$constant & 0x1f);
12343   %}
12344 
12345   ins_pipe(ialu_reg_reg_shift);
12346 %}
12347 
12348 // This pattern is automatically generated from aarch64_ad.m4
12349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12350 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12351                          iRegL src1, iRegL src2,
12352                          immI src3) %{
12353   match(Set dst (AndL src1 (RotateRight src2 src3)));
12354 
12355   ins_cost(1.9 * INSN_COST);
12356   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12357 
12358   ins_encode %{
12359     __ andr(as_Register($dst$$reg),
12360               as_Register($src1$$reg),
12361               as_Register($src2$$reg),
12362               Assembler::ROR,
12363               $src3$$constant & 0x3f);
12364   %}
12365 
12366   ins_pipe(ialu_reg_reg_shift);
12367 %}
12368 
12369 // This pattern is automatically generated from aarch64_ad.m4
12370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12371 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12372                          iRegIorL2I src1, iRegIorL2I src2,
12373                          immI src3) %{
12374   match(Set dst (XorI src1 (URShiftI src2 src3)));
12375 
12376   ins_cost(1.9 * INSN_COST);
12377   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12378 
12379   ins_encode %{
12380     __ eorw(as_Register($dst$$reg),
12381               as_Register($src1$$reg),
12382               as_Register($src2$$reg),
12383               Assembler::LSR,
12384               $src3$$constant & 0x1f);
12385   %}
12386 
12387   ins_pipe(ialu_reg_reg_shift);
12388 %}
12389 
12390 // This pattern is automatically generated from aarch64_ad.m4
12391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12392 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12393                          iRegL src1, iRegL src2,
12394                          immI src3) %{
12395   match(Set dst (XorL src1 (URShiftL src2 src3)));
12396 
12397   ins_cost(1.9 * INSN_COST);
12398   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12399 
12400   ins_encode %{
12401     __ eor(as_Register($dst$$reg),
12402               as_Register($src1$$reg),
12403               as_Register($src2$$reg),
12404               Assembler::LSR,
12405               $src3$$constant & 0x3f);
12406   %}
12407 
12408   ins_pipe(ialu_reg_reg_shift);
12409 %}
12410 
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12414                          iRegIorL2I src1, iRegIorL2I src2,
12415                          immI src3) %{
12416   match(Set dst (XorI src1 (RShiftI src2 src3)));
12417 
12418   ins_cost(1.9 * INSN_COST);
12419   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12420 
12421   ins_encode %{
12422     __ eorw(as_Register($dst$$reg),
12423               as_Register($src1$$reg),
12424               as_Register($src2$$reg),
12425               Assembler::ASR,
12426               $src3$$constant & 0x1f);
12427   %}
12428 
12429   ins_pipe(ialu_reg_reg_shift);
12430 %}
12431 
12432 // This pattern is automatically generated from aarch64_ad.m4
12433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12434 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12435                          iRegL src1, iRegL src2,
12436                          immI src3) %{
12437   match(Set dst (XorL src1 (RShiftL src2 src3)));
12438 
12439   ins_cost(1.9 * INSN_COST);
12440   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12441 
12442   ins_encode %{
12443     __ eor(as_Register($dst$$reg),
12444               as_Register($src1$$reg),
12445               as_Register($src2$$reg),
12446               Assembler::ASR,
12447               $src3$$constant & 0x3f);
12448   %}
12449 
12450   ins_pipe(ialu_reg_reg_shift);
12451 %}
12452 
12453 // This pattern is automatically generated from aarch64_ad.m4
12454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12455 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12456                          iRegIorL2I src1, iRegIorL2I src2,
12457                          immI src3) %{
12458   match(Set dst (XorI src1 (LShiftI src2 src3)));
12459 
12460   ins_cost(1.9 * INSN_COST);
12461   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12462 
12463   ins_encode %{
12464     __ eorw(as_Register($dst$$reg),
12465               as_Register($src1$$reg),
12466               as_Register($src2$$reg),
12467               Assembler::LSL,
12468               $src3$$constant & 0x1f);
12469   %}
12470 
12471   ins_pipe(ialu_reg_reg_shift);
12472 %}
12473 
12474 // This pattern is automatically generated from aarch64_ad.m4
12475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12476 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12477                          iRegL src1, iRegL src2,
12478                          immI src3) %{
12479   match(Set dst (XorL src1 (LShiftL src2 src3)));
12480 
12481   ins_cost(1.9 * INSN_COST);
12482   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12483 
12484   ins_encode %{
12485     __ eor(as_Register($dst$$reg),
12486               as_Register($src1$$reg),
12487               as_Register($src2$$reg),
12488               Assembler::LSL,
12489               $src3$$constant & 0x3f);
12490   %}
12491 
12492   ins_pipe(ialu_reg_reg_shift);
12493 %}
12494 
12495 // This pattern is automatically generated from aarch64_ad.m4
12496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12497 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12498                          iRegIorL2I src1, iRegIorL2I src2,
12499                          immI src3) %{
12500   match(Set dst (XorI src1 (RotateRight src2 src3)));
12501 
12502   ins_cost(1.9 * INSN_COST);
12503   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12504 
12505   ins_encode %{
12506     __ eorw(as_Register($dst$$reg),
12507               as_Register($src1$$reg),
12508               as_Register($src2$$reg),
12509               Assembler::ROR,
12510               $src3$$constant & 0x1f);
12511   %}
12512 
12513   ins_pipe(ialu_reg_reg_shift);
12514 %}
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12519                          iRegL src1, iRegL src2,
12520                          immI src3) %{
12521   match(Set dst (XorL src1 (RotateRight src2 src3)));
12522 
12523   ins_cost(1.9 * INSN_COST);
12524   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12525 
12526   ins_encode %{
12527     __ eor(as_Register($dst$$reg),
12528               as_Register($src1$$reg),
12529               as_Register($src2$$reg),
12530               Assembler::ROR,
12531               $src3$$constant & 0x3f);
12532   %}
12533 
12534   ins_pipe(ialu_reg_reg_shift);
12535 %}
12536 
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12540                          iRegIorL2I src1, iRegIorL2I src2,
12541                          immI src3) %{
12542   match(Set dst (OrI src1 (URShiftI src2 src3)));
12543 
12544   ins_cost(1.9 * INSN_COST);
12545   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12546 
12547   ins_encode %{
12548     __ orrw(as_Register($dst$$reg),
12549               as_Register($src1$$reg),
12550               as_Register($src2$$reg),
12551               Assembler::LSR,
12552               $src3$$constant & 0x1f);
12553   %}
12554 
12555   ins_pipe(ialu_reg_reg_shift);
12556 %}
12557 
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12561                          iRegL src1, iRegL src2,
12562                          immI src3) %{
12563   match(Set dst (OrL src1 (URShiftL src2 src3)));
12564 
12565   ins_cost(1.9 * INSN_COST);
12566   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12567 
12568   ins_encode %{
12569     __ orr(as_Register($dst$$reg),
12570               as_Register($src1$$reg),
12571               as_Register($src2$$reg),
12572               Assembler::LSR,
12573               $src3$$constant & 0x3f);
12574   %}
12575 
12576   ins_pipe(ialu_reg_reg_shift);
12577 %}
12578 
12579 // This pattern is automatically generated from aarch64_ad.m4
12580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12581 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12582                          iRegIorL2I src1, iRegIorL2I src2,
12583                          immI src3) %{
12584   match(Set dst (OrI src1 (RShiftI src2 src3)));
12585 
12586   ins_cost(1.9 * INSN_COST);
12587   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12588 
12589   ins_encode %{
12590     __ orrw(as_Register($dst$$reg),
12591               as_Register($src1$$reg),
12592               as_Register($src2$$reg),
12593               Assembler::ASR,
12594               $src3$$constant & 0x1f);
12595   %}
12596 
12597   ins_pipe(ialu_reg_reg_shift);
12598 %}
12599 
12600 // This pattern is automatically generated from aarch64_ad.m4
12601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12602 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12603                          iRegL src1, iRegL src2,
12604                          immI src3) %{
12605   match(Set dst (OrL src1 (RShiftL src2 src3)));
12606 
12607   ins_cost(1.9 * INSN_COST);
12608   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12609 
12610   ins_encode %{
12611     __ orr(as_Register($dst$$reg),
12612               as_Register($src1$$reg),
12613               as_Register($src2$$reg),
12614               Assembler::ASR,
12615               $src3$$constant & 0x3f);
12616   %}
12617 
12618   ins_pipe(ialu_reg_reg_shift);
12619 %}
12620 
12621 // This pattern is automatically generated from aarch64_ad.m4
12622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12623 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12624                          iRegIorL2I src1, iRegIorL2I src2,
12625                          immI src3) %{
12626   match(Set dst (OrI src1 (LShiftI src2 src3)));
12627 
12628   ins_cost(1.9 * INSN_COST);
12629   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12630 
12631   ins_encode %{
12632     __ orrw(as_Register($dst$$reg),
12633               as_Register($src1$$reg),
12634               as_Register($src2$$reg),
12635               Assembler::LSL,
12636               $src3$$constant & 0x1f);
12637   %}
12638 
12639   ins_pipe(ialu_reg_reg_shift);
12640 %}
12641 
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12645                          iRegL src1, iRegL src2,
12646                          immI src3) %{
12647   match(Set dst (OrL src1 (LShiftL src2 src3)));
12648 
12649   ins_cost(1.9 * INSN_COST);
12650   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12651 
12652   ins_encode %{
12653     __ orr(as_Register($dst$$reg),
12654               as_Register($src1$$reg),
12655               as_Register($src2$$reg),
12656               Assembler::LSL,
12657               $src3$$constant & 0x3f);
12658   %}
12659 
12660   ins_pipe(ialu_reg_reg_shift);
12661 %}
12662 
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12666                          iRegIorL2I src1, iRegIorL2I src2,
12667                          immI src3) %{
12668   match(Set dst (OrI src1 (RotateRight src2 src3)));
12669 
12670   ins_cost(1.9 * INSN_COST);
12671   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12672 
12673   ins_encode %{
12674     __ orrw(as_Register($dst$$reg),
12675               as_Register($src1$$reg),
12676               as_Register($src2$$reg),
12677               Assembler::ROR,
12678               $src3$$constant & 0x1f);
12679   %}
12680 
12681   ins_pipe(ialu_reg_reg_shift);
12682 %}
12683 
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12687                          iRegL src1, iRegL src2,
12688                          immI src3) %{
12689   match(Set dst (OrL src1 (RotateRight src2 src3)));
12690 
12691   ins_cost(1.9 * INSN_COST);
12692   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12693 
12694   ins_encode %{
12695     __ orr(as_Register($dst$$reg),
12696               as_Register($src1$$reg),
12697               as_Register($src2$$reg),
12698               Assembler::ROR,
12699               $src3$$constant & 0x3f);
12700   %}
12701 
12702   ins_pipe(ialu_reg_reg_shift);
12703 %}
12704 
12705 // This pattern is automatically generated from aarch64_ad.m4
12706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12707 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12708                          iRegIorL2I src1, iRegIorL2I src2,
12709                          immI src3) %{
12710   match(Set dst (AddI src1 (URShiftI src2 src3)));
12711 
12712   ins_cost(1.9 * INSN_COST);
12713   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12714 
12715   ins_encode %{
12716     __ addw(as_Register($dst$$reg),
12717               as_Register($src1$$reg),
12718               as_Register($src2$$reg),
12719               Assembler::LSR,
12720               $src3$$constant & 0x1f);
12721   %}
12722 
12723   ins_pipe(ialu_reg_reg_shift);
12724 %}
12725 
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12729                          iRegL src1, iRegL src2,
12730                          immI src3) %{
12731   match(Set dst (AddL src1 (URShiftL src2 src3)));
12732 
12733   ins_cost(1.9 * INSN_COST);
12734   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12735 
12736   ins_encode %{
12737     __ add(as_Register($dst$$reg),
12738               as_Register($src1$$reg),
12739               as_Register($src2$$reg),
12740               Assembler::LSR,
12741               $src3$$constant & 0x3f);
12742   %}
12743 
12744   ins_pipe(ialu_reg_reg_shift);
12745 %}
12746 
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12750                          iRegIorL2I src1, iRegIorL2I src2,
12751                          immI src3) %{
12752   match(Set dst (AddI src1 (RShiftI src2 src3)));
12753 
12754   ins_cost(1.9 * INSN_COST);
12755   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12756 
12757   ins_encode %{
12758     __ addw(as_Register($dst$$reg),
12759               as_Register($src1$$reg),
12760               as_Register($src2$$reg),
12761               Assembler::ASR,
12762               $src3$$constant & 0x1f);
12763   %}
12764 
12765   ins_pipe(ialu_reg_reg_shift);
12766 %}
12767 
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12771                          iRegL src1, iRegL src2,
12772                          immI src3) %{
12773   match(Set dst (AddL src1 (RShiftL src2 src3)));
12774 
12775   ins_cost(1.9 * INSN_COST);
12776   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12777 
12778   ins_encode %{
12779     __ add(as_Register($dst$$reg),
12780               as_Register($src1$$reg),
12781               as_Register($src2$$reg),
12782               Assembler::ASR,
12783               $src3$$constant & 0x3f);
12784   %}
12785 
12786   ins_pipe(ialu_reg_reg_shift);
12787 %}
12788 
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12792                          iRegIorL2I src1, iRegIorL2I src2,
12793                          immI src3) %{
12794   match(Set dst (AddI src1 (LShiftI src2 src3)));
12795 
12796   ins_cost(1.9 * INSN_COST);
12797   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12798 
12799   ins_encode %{
12800     __ addw(as_Register($dst$$reg),
12801               as_Register($src1$$reg),
12802               as_Register($src2$$reg),
12803               Assembler::LSL,
12804               $src3$$constant & 0x1f);
12805   %}
12806 
12807   ins_pipe(ialu_reg_reg_shift);
12808 %}
12809 
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12813                          iRegL src1, iRegL src2,
12814                          immI src3) %{
12815   match(Set dst (AddL src1 (LShiftL src2 src3)));
12816 
12817   ins_cost(1.9 * INSN_COST);
12818   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12819 
12820   ins_encode %{
12821     __ add(as_Register($dst$$reg),
12822               as_Register($src1$$reg),
12823               as_Register($src2$$reg),
12824               Assembler::LSL,
12825               $src3$$constant & 0x3f);
12826   %}
12827 
12828   ins_pipe(ialu_reg_reg_shift);
12829 %}
12830 
12831 // This pattern is automatically generated from aarch64_ad.m4
12832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12833 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12834                          iRegIorL2I src1, iRegIorL2I src2,
12835                          immI src3) %{
12836   match(Set dst (SubI src1 (URShiftI src2 src3)));
12837 
12838   ins_cost(1.9 * INSN_COST);
12839   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12840 
12841   ins_encode %{
12842     __ subw(as_Register($dst$$reg),
12843               as_Register($src1$$reg),
12844               as_Register($src2$$reg),
12845               Assembler::LSR,
12846               $src3$$constant & 0x1f);
12847   %}
12848 
12849   ins_pipe(ialu_reg_reg_shift);
12850 %}
12851 
12852 // This pattern is automatically generated from aarch64_ad.m4
12853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12854 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12855                          iRegL src1, iRegL src2,
12856                          immI src3) %{
12857   match(Set dst (SubL src1 (URShiftL src2 src3)));
12858 
12859   ins_cost(1.9 * INSN_COST);
12860   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12861 
12862   ins_encode %{
12863     __ sub(as_Register($dst$$reg),
12864               as_Register($src1$$reg),
12865               as_Register($src2$$reg),
12866               Assembler::LSR,
12867               $src3$$constant & 0x3f);
12868   %}
12869 
12870   ins_pipe(ialu_reg_reg_shift);
12871 %}
12872 
12873 // This pattern is automatically generated from aarch64_ad.m4
12874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12875 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12876                          iRegIorL2I src1, iRegIorL2I src2,
12877                          immI src3) %{
12878   match(Set dst (SubI src1 (RShiftI src2 src3)));
12879 
12880   ins_cost(1.9 * INSN_COST);
12881   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12882 
12883   ins_encode %{
12884     __ subw(as_Register($dst$$reg),
12885               as_Register($src1$$reg),
12886               as_Register($src2$$reg),
12887               Assembler::ASR,
12888               $src3$$constant & 0x1f);
12889   %}
12890 
12891   ins_pipe(ialu_reg_reg_shift);
12892 %}
12893 
12894 // This pattern is automatically generated from aarch64_ad.m4
12895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12897                          iRegL src1, iRegL src2,
12898                          immI src3) %{
12899   match(Set dst (SubL src1 (RShiftL src2 src3)));
12900 
12901   ins_cost(1.9 * INSN_COST);
12902   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12903 
12904   ins_encode %{
12905     __ sub(as_Register($dst$$reg),
12906               as_Register($src1$$reg),
12907               as_Register($src2$$reg),
12908               Assembler::ASR,
12909               $src3$$constant & 0x3f);
12910   %}
12911 
12912   ins_pipe(ialu_reg_reg_shift);
12913 %}
12914 
12915 // This pattern is automatically generated from aarch64_ad.m4
12916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12917 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12918                          iRegIorL2I src1, iRegIorL2I src2,
12919                          immI src3) %{
12920   match(Set dst (SubI src1 (LShiftI src2 src3)));
12921 
12922   ins_cost(1.9 * INSN_COST);
12923   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12924 
12925   ins_encode %{
12926     __ subw(as_Register($dst$$reg),
12927               as_Register($src1$$reg),
12928               as_Register($src2$$reg),
12929               Assembler::LSL,
12930               $src3$$constant & 0x1f);
12931   %}
12932 
12933   ins_pipe(ialu_reg_reg_shift);
12934 %}
12935 
12936 // This pattern is automatically generated from aarch64_ad.m4
12937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12938 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12939                          iRegL src1, iRegL src2,
12940                          immI src3) %{
12941   match(Set dst (SubL src1 (LShiftL src2 src3)));
12942 
12943   ins_cost(1.9 * INSN_COST);
12944   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12945 
12946   ins_encode %{
12947     __ sub(as_Register($dst$$reg),
12948               as_Register($src1$$reg),
12949               as_Register($src2$$reg),
12950               Assembler::LSL,
12951               $src3$$constant & 0x3f);
12952   %}
12953 
12954   ins_pipe(ialu_reg_reg_shift);
12955 %}
12956 
12957 // This pattern is automatically generated from aarch64_ad.m4
12958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12959 
12960 // Shift Left followed by Shift Right.
12961 // This idiom is used by the compiler for the i2b bytecode etc.
12962 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12963 %{
12964   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12965   ins_cost(INSN_COST * 2);
12966   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12967   ins_encode %{
12968     int lshift = $lshift_count$$constant & 63;
12969     int rshift = $rshift_count$$constant & 63;
12970     int s = 63 - lshift;
12971     int r = (rshift - lshift) & 63;
12972     __ sbfm(as_Register($dst$$reg),
12973             as_Register($src$$reg),
12974             r, s);
12975   %}
12976 
12977   ins_pipe(ialu_reg_shift);
12978 %}
12979 
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 
12983 // Shift Left followed by Shift Right.
12984 // This idiom is used by the compiler for the i2b bytecode etc.
12985 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12986 %{
12987   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12988   ins_cost(INSN_COST * 2);
12989   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12990   ins_encode %{
12991     int lshift = $lshift_count$$constant & 31;
12992     int rshift = $rshift_count$$constant & 31;
12993     int s = 31 - lshift;
12994     int r = (rshift - lshift) & 31;
12995     __ sbfmw(as_Register($dst$$reg),
12996             as_Register($src$$reg),
12997             r, s);
12998   %}
12999 
13000   ins_pipe(ialu_reg_shift);
13001 %}
13002 
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 
13006 // Shift Left followed by Shift Right.
13007 // This idiom is used by the compiler for the i2b bytecode etc.
13008 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13009 %{
13010   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13011   ins_cost(INSN_COST * 2);
13012   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13013   ins_encode %{
13014     int lshift = $lshift_count$$constant & 63;
13015     int rshift = $rshift_count$$constant & 63;
13016     int s = 63 - lshift;
13017     int r = (rshift - lshift) & 63;
13018     __ ubfm(as_Register($dst$$reg),
13019             as_Register($src$$reg),
13020             r, s);
13021   %}
13022 
13023   ins_pipe(ialu_reg_shift);
13024 %}
13025 
13026 // This pattern is automatically generated from aarch64_ad.m4
13027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13028 
13029 // Shift Left followed by Shift Right.
13030 // This idiom is used by the compiler for the i2b bytecode etc.
13031 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13032 %{
13033   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13034   ins_cost(INSN_COST * 2);
13035   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13036   ins_encode %{
13037     int lshift = $lshift_count$$constant & 31;
13038     int rshift = $rshift_count$$constant & 31;
13039     int s = 31 - lshift;
13040     int r = (rshift - lshift) & 31;
13041     __ ubfmw(as_Register($dst$$reg),
13042             as_Register($src$$reg),
13043             r, s);
13044   %}
13045 
13046   ins_pipe(ialu_reg_shift);
13047 %}
13048 
13049 // Bitfield extract with shift & mask
13050 
13051 // This pattern is automatically generated from aarch64_ad.m4
13052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13053 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13054 %{
13055   match(Set dst (AndI (URShiftI src rshift) mask));
13056   // Make sure we are not going to exceed what ubfxw can do.
13057   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13058 
13059   ins_cost(INSN_COST);
13060   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13061   ins_encode %{
13062     int rshift = $rshift$$constant & 31;
13063     intptr_t mask = $mask$$constant;
13064     int width = exact_log2(mask+1);
13065     __ ubfxw(as_Register($dst$$reg),
13066             as_Register($src$$reg), rshift, width);
13067   %}
13068   ins_pipe(ialu_reg_shift);
13069 %}
13070 
13071 // This pattern is automatically generated from aarch64_ad.m4
13072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13073 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13074 %{
13075   match(Set dst (AndL (URShiftL src rshift) mask));
13076   // Make sure we are not going to exceed what ubfx can do.
13077   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13078 
13079   ins_cost(INSN_COST);
13080   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13081   ins_encode %{
13082     int rshift = $rshift$$constant & 63;
13083     intptr_t mask = $mask$$constant;
13084     int width = exact_log2_long(mask+1);
13085     __ ubfx(as_Register($dst$$reg),
13086             as_Register($src$$reg), rshift, width);
13087   %}
13088   ins_pipe(ialu_reg_shift);
13089 %}
13090 
13091 
13092 // This pattern is automatically generated from aarch64_ad.m4
13093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13094 
13095 // We can use ubfx when extending an And with a mask when we know mask
13096 // is positive.  We know that because immI_bitmask guarantees it.
13097 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13098 %{
13099   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13100   // Make sure we are not going to exceed what ubfxw can do.
13101   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13102 
13103   ins_cost(INSN_COST * 2);
13104   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13105   ins_encode %{
13106     int rshift = $rshift$$constant & 31;
13107     intptr_t mask = $mask$$constant;
13108     int width = exact_log2(mask+1);
13109     __ ubfx(as_Register($dst$$reg),
13110             as_Register($src$$reg), rshift, width);
13111   %}
13112   ins_pipe(ialu_reg_shift);
13113 %}
13114 
13115 
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 
13119 // We can use ubfiz when masking by a positive number and then left shifting the result.
13120 // We know that the mask is positive because immI_bitmask guarantees it.
13121 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13122 %{
13123   match(Set dst (LShiftI (AndI src mask) lshift));
13124   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13125 
13126   ins_cost(INSN_COST);
13127   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13128   ins_encode %{
13129     int lshift = $lshift$$constant & 31;
13130     intptr_t mask = $mask$$constant;
13131     int width = exact_log2(mask+1);
13132     __ ubfizw(as_Register($dst$$reg),
13133           as_Register($src$$reg), lshift, width);
13134   %}
13135   ins_pipe(ialu_reg_shift);
13136 %}
13137 
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 
13141 // We can use ubfiz when masking by a positive number and then left shifting the result.
13142 // We know that the mask is positive because immL_bitmask guarantees it.
13143 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13144 %{
13145   match(Set dst (LShiftL (AndL src mask) lshift));
13146   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13147 
13148   ins_cost(INSN_COST);
13149   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13150   ins_encode %{
13151     int lshift = $lshift$$constant & 63;
13152     intptr_t mask = $mask$$constant;
13153     int width = exact_log2_long(mask+1);
13154     __ ubfiz(as_Register($dst$$reg),
13155           as_Register($src$$reg), lshift, width);
13156   %}
13157   ins_pipe(ialu_reg_shift);
13158 %}
13159 
13160 // This pattern is automatically generated from aarch64_ad.m4
13161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13162 
13163 // We can use ubfiz when masking by a positive number and then left shifting the result.
13164 // We know that the mask is positive because immI_bitmask guarantees it.
13165 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13166 %{
13167   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13168   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13169 
13170   ins_cost(INSN_COST);
13171   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13172   ins_encode %{
13173     int lshift = $lshift$$constant & 31;
13174     intptr_t mask = $mask$$constant;
13175     int width = exact_log2(mask+1);
13176     __ ubfizw(as_Register($dst$$reg),
13177           as_Register($src$$reg), lshift, width);
13178   %}
13179   ins_pipe(ialu_reg_shift);
13180 %}
13181 
13182 // This pattern is automatically generated from aarch64_ad.m4
13183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13184 
13185 // We can use ubfiz when masking by a positive number and then left shifting the result.
13186 // We know that the mask is positive because immL_bitmask guarantees it.
13187 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13188 %{
13189   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13190   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13191 
13192   ins_cost(INSN_COST);
13193   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13194   ins_encode %{
13195     int lshift = $lshift$$constant & 63;
13196     intptr_t mask = $mask$$constant;
13197     int width = exact_log2_long(mask+1);
13198     __ ubfiz(as_Register($dst$$reg),
13199           as_Register($src$$reg), lshift, width);
13200   %}
13201   ins_pipe(ialu_reg_shift);
13202 %}
13203 
13204 
13205 // This pattern is automatically generated from aarch64_ad.m4
13206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13207 
13208 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13209 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13210 %{
13211   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13212   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13213 
13214   ins_cost(INSN_COST);
13215   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13216   ins_encode %{
13217     int lshift = $lshift$$constant & 63;
13218     intptr_t mask = $mask$$constant;
13219     int width = exact_log2(mask+1);
13220     __ ubfiz(as_Register($dst$$reg),
13221              as_Register($src$$reg), lshift, width);
13222   %}
13223   ins_pipe(ialu_reg_shift);
13224 %}
13225 
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 
13229 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13230 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13231 %{
13232   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13233   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13234 
13235   ins_cost(INSN_COST);
13236   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13237   ins_encode %{
13238     int lshift = $lshift$$constant & 31;
13239     intptr_t mask = $mask$$constant;
13240     int width = exact_log2(mask+1);
13241     __ ubfiz(as_Register($dst$$reg),
13242              as_Register($src$$reg), lshift, width);
13243   %}
13244   ins_pipe(ialu_reg_shift);
13245 %}
13246 
13247 // This pattern is automatically generated from aarch64_ad.m4
13248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13249 
13250 // Can skip int2long conversions after AND with small bitmask
13251 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13252 %{
13253   match(Set dst (ConvI2L (AndI src msk)));
13254   ins_cost(INSN_COST);
13255   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13256   ins_encode %{
13257     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13258   %}
13259   ins_pipe(ialu_reg_shift);
13260 %}
13261 
13262 
13263 // Rotations
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13267 %{
13268   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13269   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13270 
13271   ins_cost(INSN_COST);
13272   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13273 
13274   ins_encode %{
13275     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13276             $rshift$$constant & 63);
13277   %}
13278   ins_pipe(ialu_reg_reg_extr);
13279 %}
13280 
13281 
13282 // This pattern is automatically generated from aarch64_ad.m4
13283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13284 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13285 %{
13286   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13287   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13288 
13289   ins_cost(INSN_COST);
13290   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13291 
13292   ins_encode %{
13293     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13294             $rshift$$constant & 31);
13295   %}
13296   ins_pipe(ialu_reg_reg_extr);
13297 %}
13298 
13299 
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13303 %{
13304   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13305   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13306 
13307   ins_cost(INSN_COST);
13308   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13309 
13310   ins_encode %{
13311     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13312             $rshift$$constant & 63);
13313   %}
13314   ins_pipe(ialu_reg_reg_extr);
13315 %}
13316 
13317 
13318 // This pattern is automatically generated from aarch64_ad.m4
13319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13320 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13321 %{
13322   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13323   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13324 
13325   ins_cost(INSN_COST);
13326   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13327 
13328   ins_encode %{
13329     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13330             $rshift$$constant & 31);
13331   %}
13332   ins_pipe(ialu_reg_reg_extr);
13333 %}
13334 
13335 
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13339 %{
13340   match(Set dst (RotateRight src shift));
13341 
13342   ins_cost(INSN_COST);
13343   format %{ "ror    $dst, $src, $shift" %}
13344 
13345   ins_encode %{
13346      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13347                $shift$$constant & 0x1f);
13348   %}
13349   ins_pipe(ialu_reg_reg_vshift);
13350 %}
13351 
13352 // This pattern is automatically generated from aarch64_ad.m4
13353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13354 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13355 %{
13356   match(Set dst (RotateRight src shift));
13357 
13358   ins_cost(INSN_COST);
13359   format %{ "ror    $dst, $src, $shift" %}
13360 
13361   ins_encode %{
13362      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13363                $shift$$constant & 0x3f);
13364   %}
13365   ins_pipe(ialu_reg_reg_vshift);
13366 %}
13367 
13368 // This pattern is automatically generated from aarch64_ad.m4
13369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13370 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13371 %{
13372   match(Set dst (RotateRight src shift));
13373 
13374   ins_cost(INSN_COST);
13375   format %{ "ror    $dst, $src, $shift" %}
13376 
13377   ins_encode %{
13378      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13379   %}
13380   ins_pipe(ialu_reg_reg_vshift);
13381 %}
13382 
13383 // This pattern is automatically generated from aarch64_ad.m4
13384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13385 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13386 %{
13387   match(Set dst (RotateRight src shift));
13388 
13389   ins_cost(INSN_COST);
13390   format %{ "ror    $dst, $src, $shift" %}
13391 
13392   ins_encode %{
13393      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13394   %}
13395   ins_pipe(ialu_reg_reg_vshift);
13396 %}
13397 
13398 // This pattern is automatically generated from aarch64_ad.m4
13399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13400 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13401 %{
13402   match(Set dst (RotateLeft src shift));
13403 
13404   ins_cost(INSN_COST);
13405   format %{ "rol    $dst, $src, $shift" %}
13406 
13407   ins_encode %{
13408      __ subw(rscratch1, zr, as_Register($shift$$reg));
13409      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13410   %}
13411   ins_pipe(ialu_reg_reg_vshift);
13412 %}
13413 
13414 // This pattern is automatically generated from aarch64_ad.m4
13415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13416 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13417 %{
13418   match(Set dst (RotateLeft src shift));
13419 
13420   ins_cost(INSN_COST);
13421   format %{ "rol    $dst, $src, $shift" %}
13422 
13423   ins_encode %{
13424      __ subw(rscratch1, zr, as_Register($shift$$reg));
13425      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13426   %}
13427   ins_pipe(ialu_reg_reg_vshift);
13428 %}
13429 
13430 
13431 // Add/subtract (extended)
13432 
13433 // This pattern is automatically generated from aarch64_ad.m4
13434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13435 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13436 %{
13437   match(Set dst (AddL src1 (ConvI2L src2)));
13438   ins_cost(INSN_COST);
13439   format %{ "add  $dst, $src1, $src2, sxtw" %}
13440 
13441    ins_encode %{
13442      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13443             as_Register($src2$$reg), ext::sxtw);
13444    %}
13445   ins_pipe(ialu_reg_reg);
13446 %}
13447 
13448 // This pattern is automatically generated from aarch64_ad.m4
13449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13450 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13451 %{
13452   match(Set dst (SubL src1 (ConvI2L src2)));
13453   ins_cost(INSN_COST);
13454   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13455 
13456    ins_encode %{
13457      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13458             as_Register($src2$$reg), ext::sxtw);
13459    %}
13460   ins_pipe(ialu_reg_reg);
13461 %}
13462 
13463 // This pattern is automatically generated from aarch64_ad.m4
13464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13465 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13466 %{
13467   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13468   ins_cost(INSN_COST);
13469   format %{ "add  $dst, $src1, $src2, sxth" %}
13470 
13471    ins_encode %{
13472      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13473             as_Register($src2$$reg), ext::sxth);
13474    %}
13475   ins_pipe(ialu_reg_reg);
13476 %}
13477 
13478 // This pattern is automatically generated from aarch64_ad.m4
13479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13480 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13481 %{
13482   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13483   ins_cost(INSN_COST);
13484   format %{ "add  $dst, $src1, $src2, sxtb" %}
13485 
13486    ins_encode %{
13487      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13488             as_Register($src2$$reg), ext::sxtb);
13489    %}
13490   ins_pipe(ialu_reg_reg);
13491 %}
13492 
13493 // This pattern is automatically generated from aarch64_ad.m4
13494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13495 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13496 %{
13497   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13498   ins_cost(INSN_COST);
13499   format %{ "add  $dst, $src1, $src2, uxtb" %}
13500 
13501    ins_encode %{
13502      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13503             as_Register($src2$$reg), ext::uxtb);
13504    %}
13505   ins_pipe(ialu_reg_reg);
13506 %}
13507 
13508 // This pattern is automatically generated from aarch64_ad.m4
13509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13510 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13511 %{
13512   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13513   ins_cost(INSN_COST);
13514   format %{ "add  $dst, $src1, $src2, sxth" %}
13515 
13516    ins_encode %{
13517      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13518             as_Register($src2$$reg), ext::sxth);
13519    %}
13520   ins_pipe(ialu_reg_reg);
13521 %}
13522 
13523 // This pattern is automatically generated from aarch64_ad.m4
13524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13525 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13526 %{
13527   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13528   ins_cost(INSN_COST);
13529   format %{ "add  $dst, $src1, $src2, sxtw" %}
13530 
13531    ins_encode %{
13532      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13533             as_Register($src2$$reg), ext::sxtw);
13534    %}
13535   ins_pipe(ialu_reg_reg);
13536 %}
13537 
13538 // This pattern is automatically generated from aarch64_ad.m4
13539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13540 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13541 %{
13542   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13543   ins_cost(INSN_COST);
13544   format %{ "add  $dst, $src1, $src2, sxtb" %}
13545 
13546    ins_encode %{
13547      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13548             as_Register($src2$$reg), ext::sxtb);
13549    %}
13550   ins_pipe(ialu_reg_reg);
13551 %}
13552 
13553 // This pattern is automatically generated from aarch64_ad.m4
13554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13555 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13556 %{
13557   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13558   ins_cost(INSN_COST);
13559   format %{ "add  $dst, $src1, $src2, uxtb" %}
13560 
13561    ins_encode %{
13562      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13563             as_Register($src2$$reg), ext::uxtb);
13564    %}
13565   ins_pipe(ialu_reg_reg);
13566 %}
13567 
13568 // This pattern is automatically generated from aarch64_ad.m4
13569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13570 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13571 %{
13572   match(Set dst (AddI src1 (AndI src2 mask)));
13573   ins_cost(INSN_COST);
13574   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13575 
13576    ins_encode %{
13577      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13578             as_Register($src2$$reg), ext::uxtb);
13579    %}
13580   ins_pipe(ialu_reg_reg);
13581 %}
13582 
13583 // This pattern is automatically generated from aarch64_ad.m4
13584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13585 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13586 %{
13587   match(Set dst (AddI src1 (AndI src2 mask)));
13588   ins_cost(INSN_COST);
13589   format %{ "addw  $dst, $src1, $src2, uxth" %}
13590 
13591    ins_encode %{
13592      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13593             as_Register($src2$$reg), ext::uxth);
13594    %}
13595   ins_pipe(ialu_reg_reg);
13596 %}
13597 
13598 // This pattern is automatically generated from aarch64_ad.m4
13599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13600 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13601 %{
13602   match(Set dst (AddL src1 (AndL src2 mask)));
13603   ins_cost(INSN_COST);
13604   format %{ "add  $dst, $src1, $src2, uxtb" %}
13605 
13606    ins_encode %{
13607      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13608             as_Register($src2$$reg), ext::uxtb);
13609    %}
13610   ins_pipe(ialu_reg_reg);
13611 %}
13612 
13613 // This pattern is automatically generated from aarch64_ad.m4
13614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13615 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13616 %{
13617   match(Set dst (AddL src1 (AndL src2 mask)));
13618   ins_cost(INSN_COST);
13619   format %{ "add  $dst, $src1, $src2, uxth" %}
13620 
13621    ins_encode %{
13622      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13623             as_Register($src2$$reg), ext::uxth);
13624    %}
13625   ins_pipe(ialu_reg_reg);
13626 %}
13627 
13628 // This pattern is automatically generated from aarch64_ad.m4
13629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13630 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13631 %{
13632   match(Set dst (AddL src1 (AndL src2 mask)));
13633   ins_cost(INSN_COST);
13634   format %{ "add  $dst, $src1, $src2, uxtw" %}
13635 
13636    ins_encode %{
13637      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13638             as_Register($src2$$reg), ext::uxtw);
13639    %}
13640   ins_pipe(ialu_reg_reg);
13641 %}
13642 
13643 // This pattern is automatically generated from aarch64_ad.m4
13644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13645 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13646 %{
13647   match(Set dst (SubI src1 (AndI src2 mask)));
13648   ins_cost(INSN_COST);
13649   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13650 
13651    ins_encode %{
13652      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13653             as_Register($src2$$reg), ext::uxtb);
13654    %}
13655   ins_pipe(ialu_reg_reg);
13656 %}
13657 
13658 // This pattern is automatically generated from aarch64_ad.m4
13659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13660 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13661 %{
13662   match(Set dst (SubI src1 (AndI src2 mask)));
13663   ins_cost(INSN_COST);
13664   format %{ "subw  $dst, $src1, $src2, uxth" %}
13665 
13666    ins_encode %{
13667      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13668             as_Register($src2$$reg), ext::uxth);
13669    %}
13670   ins_pipe(ialu_reg_reg);
13671 %}
13672 
13673 // This pattern is automatically generated from aarch64_ad.m4
13674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13675 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13676 %{
13677   match(Set dst (SubL src1 (AndL src2 mask)));
13678   ins_cost(INSN_COST);
13679   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13680 
13681    ins_encode %{
13682      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13683             as_Register($src2$$reg), ext::uxtb);
13684    %}
13685   ins_pipe(ialu_reg_reg);
13686 %}
13687 
13688 // This pattern is automatically generated from aarch64_ad.m4
13689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13690 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13691 %{
13692   match(Set dst (SubL src1 (AndL src2 mask)));
13693   ins_cost(INSN_COST);
13694   format %{ "sub  $dst, $src1, $src2, uxth" %}
13695 
13696    ins_encode %{
13697      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13698             as_Register($src2$$reg), ext::uxth);
13699    %}
13700   ins_pipe(ialu_reg_reg);
13701 %}
13702 
13703 // This pattern is automatically generated from aarch64_ad.m4
13704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13705 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13706 %{
13707   match(Set dst (SubL src1 (AndL src2 mask)));
13708   ins_cost(INSN_COST);
13709   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13710 
13711    ins_encode %{
13712      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13713             as_Register($src2$$reg), ext::uxtw);
13714    %}
13715   ins_pipe(ialu_reg_reg);
13716 %}
13717 
13718 
13719 // This pattern is automatically generated from aarch64_ad.m4
13720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13721 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13722 %{
13723   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13724   ins_cost(1.9 * INSN_COST);
13725   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13726 
13727    ins_encode %{
13728      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13729             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13730    %}
13731   ins_pipe(ialu_reg_reg_shift);
13732 %}
13733 
13734 // This pattern is automatically generated from aarch64_ad.m4
13735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13736 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13737 %{
13738   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13739   ins_cost(1.9 * INSN_COST);
13740   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13741 
13742    ins_encode %{
13743      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13744             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13745    %}
13746   ins_pipe(ialu_reg_reg_shift);
13747 %}
13748 
13749 // This pattern is automatically generated from aarch64_ad.m4
13750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13751 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13752 %{
13753   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13754   ins_cost(1.9 * INSN_COST);
13755   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13756 
13757    ins_encode %{
13758      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13759             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13760    %}
13761   ins_pipe(ialu_reg_reg_shift);
13762 %}
13763 
13764 // This pattern is automatically generated from aarch64_ad.m4
13765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13766 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13767 %{
13768   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13769   ins_cost(1.9 * INSN_COST);
13770   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13771 
13772    ins_encode %{
13773      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13774             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13775    %}
13776   ins_pipe(ialu_reg_reg_shift);
13777 %}
13778 
13779 // This pattern is automatically generated from aarch64_ad.m4
13780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13781 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13782 %{
13783   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13784   ins_cost(1.9 * INSN_COST);
13785   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13786 
13787    ins_encode %{
13788      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13789             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13790    %}
13791   ins_pipe(ialu_reg_reg_shift);
13792 %}
13793 
13794 // This pattern is automatically generated from aarch64_ad.m4
13795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13796 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13797 %{
13798   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13799   ins_cost(1.9 * INSN_COST);
13800   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13801 
13802    ins_encode %{
13803      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13804             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13805    %}
13806   ins_pipe(ialu_reg_reg_shift);
13807 %}
13808 
13809 // This pattern is automatically generated from aarch64_ad.m4
13810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13811 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13812 %{
13813   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13814   ins_cost(1.9 * INSN_COST);
13815   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13816 
13817    ins_encode %{
13818      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13819             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13820    %}
13821   ins_pipe(ialu_reg_reg_shift);
13822 %}
13823 
13824 // This pattern is automatically generated from aarch64_ad.m4
13825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13826 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13827 %{
13828   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13829   ins_cost(1.9 * INSN_COST);
13830   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13831 
13832    ins_encode %{
13833      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13834             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13835    %}
13836   ins_pipe(ialu_reg_reg_shift);
13837 %}
13838 
13839 // This pattern is automatically generated from aarch64_ad.m4
13840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13841 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13842 %{
13843   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13844   ins_cost(1.9 * INSN_COST);
13845   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13846 
13847    ins_encode %{
13848      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13849             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13850    %}
13851   ins_pipe(ialu_reg_reg_shift);
13852 %}
13853 
13854 // This pattern is automatically generated from aarch64_ad.m4
13855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13856 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13857 %{
13858   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13859   ins_cost(1.9 * INSN_COST);
13860   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13861 
13862    ins_encode %{
13863      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13864             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13865    %}
13866   ins_pipe(ialu_reg_reg_shift);
13867 %}
13868 
13869 // This pattern is automatically generated from aarch64_ad.m4
13870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13871 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13872 %{
13873   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13874   ins_cost(1.9 * INSN_COST);
13875   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13876 
13877    ins_encode %{
13878      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13879             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13880    %}
13881   ins_pipe(ialu_reg_reg_shift);
13882 %}
13883 
13884 // This pattern is automatically generated from aarch64_ad.m4
13885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13886 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13887 %{
13888   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13889   ins_cost(1.9 * INSN_COST);
13890   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13891 
13892    ins_encode %{
13893      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13894             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13895    %}
13896   ins_pipe(ialu_reg_reg_shift);
13897 %}
13898 
13899 // This pattern is automatically generated from aarch64_ad.m4
13900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13901 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13902 %{
13903   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13904   ins_cost(1.9 * INSN_COST);
13905   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13906 
13907    ins_encode %{
13908      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13909             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13910    %}
13911   ins_pipe(ialu_reg_reg_shift);
13912 %}
13913 
13914 // This pattern is automatically generated from aarch64_ad.m4
13915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13916 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13917 %{
13918   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13919   ins_cost(1.9 * INSN_COST);
13920   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13921 
13922    ins_encode %{
13923      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13924             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13925    %}
13926   ins_pipe(ialu_reg_reg_shift);
13927 %}
13928 
13929 // This pattern is automatically generated from aarch64_ad.m4
13930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13931 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13932 %{
13933   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13934   ins_cost(1.9 * INSN_COST);
13935   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13936 
13937    ins_encode %{
13938      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13939             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13940    %}
13941   ins_pipe(ialu_reg_reg_shift);
13942 %}
13943 
13944 // This pattern is automatically generated from aarch64_ad.m4
13945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13946 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13947 %{
13948   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13949   ins_cost(1.9 * INSN_COST);
13950   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13951 
13952    ins_encode %{
13953      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13954             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13955    %}
13956   ins_pipe(ialu_reg_reg_shift);
13957 %}
13958 
13959 // This pattern is automatically generated from aarch64_ad.m4
13960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13961 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13962 %{
13963   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13964   ins_cost(1.9 * INSN_COST);
13965   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13966 
13967    ins_encode %{
13968      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13969             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13970    %}
13971   ins_pipe(ialu_reg_reg_shift);
13972 %}
13973 
13974 // This pattern is automatically generated from aarch64_ad.m4
13975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13976 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13977 %{
13978   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13979   ins_cost(1.9 * INSN_COST);
13980   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13981 
13982    ins_encode %{
13983      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13984             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13985    %}
13986   ins_pipe(ialu_reg_reg_shift);
13987 %}
13988 
13989 // This pattern is automatically generated from aarch64_ad.m4
13990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13991 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13992 %{
13993   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13994   ins_cost(1.9 * INSN_COST);
13995   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13996 
13997    ins_encode %{
13998      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13999             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14000    %}
14001   ins_pipe(ialu_reg_reg_shift);
14002 %}
14003 
14004 // This pattern is automatically generated from aarch64_ad.m4
14005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14006 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14007 %{
14008   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14009   ins_cost(1.9 * INSN_COST);
14010   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
14011 
14012    ins_encode %{
14013      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14014             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14015    %}
14016   ins_pipe(ialu_reg_reg_shift);
14017 %}
14018 
14019 // This pattern is automatically generated from aarch64_ad.m4
14020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14021 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14022 %{
14023   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14024   ins_cost(1.9 * INSN_COST);
14025   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14026 
14027    ins_encode %{
14028      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14029             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14030    %}
14031   ins_pipe(ialu_reg_reg_shift);
14032 %}
14033 
14034 // This pattern is automatically generated from aarch64_ad.m4
14035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14036 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14037 %{
14038   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14039   ins_cost(1.9 * INSN_COST);
14040   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14041 
14042    ins_encode %{
14043      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14044             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14045    %}
14046   ins_pipe(ialu_reg_reg_shift);
14047 %}
14048 
14049 
14050 
14051 // END This section of the file is automatically generated. Do not edit --------------
14052 
14053 
14054 // ============================================================================
14055 // Floating Point Arithmetic Instructions
14056 
14057 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14058   match(Set dst (AddF src1 src2));
14059 
14060   ins_cost(INSN_COST * 5);
14061   format %{ "fadds   $dst, $src1, $src2" %}
14062 
14063   ins_encode %{
14064     __ fadds(as_FloatRegister($dst$$reg),
14065              as_FloatRegister($src1$$reg),
14066              as_FloatRegister($src2$$reg));
14067   %}
14068 
14069   ins_pipe(fp_dop_reg_reg_s);
14070 %}
14071 
14072 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14073   match(Set dst (AddD src1 src2));
14074 
14075   ins_cost(INSN_COST * 5);
14076   format %{ "faddd   $dst, $src1, $src2" %}
14077 
14078   ins_encode %{
14079     __ faddd(as_FloatRegister($dst$$reg),
14080              as_FloatRegister($src1$$reg),
14081              as_FloatRegister($src2$$reg));
14082   %}
14083 
14084   ins_pipe(fp_dop_reg_reg_d);
14085 %}
14086 
14087 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14088   match(Set dst (SubF src1 src2));
14089 
14090   ins_cost(INSN_COST * 5);
14091   format %{ "fsubs   $dst, $src1, $src2" %}
14092 
14093   ins_encode %{
14094     __ fsubs(as_FloatRegister($dst$$reg),
14095              as_FloatRegister($src1$$reg),
14096              as_FloatRegister($src2$$reg));
14097   %}
14098 
14099   ins_pipe(fp_dop_reg_reg_s);
14100 %}
14101 
14102 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14103   match(Set dst (SubD src1 src2));
14104 
14105   ins_cost(INSN_COST * 5);
14106   format %{ "fsubd   $dst, $src1, $src2" %}
14107 
14108   ins_encode %{
14109     __ fsubd(as_FloatRegister($dst$$reg),
14110              as_FloatRegister($src1$$reg),
14111              as_FloatRegister($src2$$reg));
14112   %}
14113 
14114   ins_pipe(fp_dop_reg_reg_d);
14115 %}
14116 
14117 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14118   match(Set dst (MulF src1 src2));
14119 
14120   ins_cost(INSN_COST * 6);
14121   format %{ "fmuls   $dst, $src1, $src2" %}
14122 
14123   ins_encode %{
14124     __ fmuls(as_FloatRegister($dst$$reg),
14125              as_FloatRegister($src1$$reg),
14126              as_FloatRegister($src2$$reg));
14127   %}
14128 
14129   ins_pipe(fp_dop_reg_reg_s);
14130 %}
14131 
14132 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14133   match(Set dst (MulD src1 src2));
14134 
14135   ins_cost(INSN_COST * 6);
14136   format %{ "fmuld   $dst, $src1, $src2" %}
14137 
14138   ins_encode %{
14139     __ fmuld(as_FloatRegister($dst$$reg),
14140              as_FloatRegister($src1$$reg),
14141              as_FloatRegister($src2$$reg));
14142   %}
14143 
14144   ins_pipe(fp_dop_reg_reg_d);
14145 %}
14146 
14147 // src1 * src2 + src3
14148 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14149   predicate(UseFMA);
14150   match(Set dst (FmaF src3 (Binary src1 src2)));
14151 
14152   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14153 
14154   ins_encode %{
14155     __ fmadds(as_FloatRegister($dst$$reg),
14156              as_FloatRegister($src1$$reg),
14157              as_FloatRegister($src2$$reg),
14158              as_FloatRegister($src3$$reg));
14159   %}
14160 
14161   ins_pipe(pipe_class_default);
14162 %}
14163 
14164 // src1 * src2 + src3
14165 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14166   predicate(UseFMA);
14167   match(Set dst (FmaD src3 (Binary src1 src2)));
14168 
14169   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14170 
14171   ins_encode %{
14172     __ fmaddd(as_FloatRegister($dst$$reg),
14173              as_FloatRegister($src1$$reg),
14174              as_FloatRegister($src2$$reg),
14175              as_FloatRegister($src3$$reg));
14176   %}
14177 
14178   ins_pipe(pipe_class_default);
14179 %}
14180 
14181 // -src1 * src2 + src3
14182 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14183   predicate(UseFMA);
14184   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14185   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14186 
14187   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14188 
14189   ins_encode %{
14190     __ fmsubs(as_FloatRegister($dst$$reg),
14191               as_FloatRegister($src1$$reg),
14192               as_FloatRegister($src2$$reg),
14193               as_FloatRegister($src3$$reg));
14194   %}
14195 
14196   ins_pipe(pipe_class_default);
14197 %}
14198 
14199 // -src1 * src2 + src3
14200 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14201   predicate(UseFMA);
14202   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14203   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14204 
14205   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14206 
14207   ins_encode %{
14208     __ fmsubd(as_FloatRegister($dst$$reg),
14209               as_FloatRegister($src1$$reg),
14210               as_FloatRegister($src2$$reg),
14211               as_FloatRegister($src3$$reg));
14212   %}
14213 
14214   ins_pipe(pipe_class_default);
14215 %}
14216 
14217 // -src1 * src2 - src3
14218 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14219   predicate(UseFMA);
14220   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14221   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14222 
14223   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14224 
14225   ins_encode %{
14226     __ fnmadds(as_FloatRegister($dst$$reg),
14227                as_FloatRegister($src1$$reg),
14228                as_FloatRegister($src2$$reg),
14229                as_FloatRegister($src3$$reg));
14230   %}
14231 
14232   ins_pipe(pipe_class_default);
14233 %}
14234 
14235 // -src1 * src2 - src3
14236 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14237   predicate(UseFMA);
14238   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14239   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14240 
14241   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14242 
14243   ins_encode %{
14244     __ fnmaddd(as_FloatRegister($dst$$reg),
14245                as_FloatRegister($src1$$reg),
14246                as_FloatRegister($src2$$reg),
14247                as_FloatRegister($src3$$reg));
14248   %}
14249 
14250   ins_pipe(pipe_class_default);
14251 %}
14252 
14253 // src1 * src2 - src3
14254 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14255   predicate(UseFMA);
14256   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14257 
14258   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14259 
14260   ins_encode %{
14261     __ fnmsubs(as_FloatRegister($dst$$reg),
14262                as_FloatRegister($src1$$reg),
14263                as_FloatRegister($src2$$reg),
14264                as_FloatRegister($src3$$reg));
14265   %}
14266 
14267   ins_pipe(pipe_class_default);
14268 %}
14269 
14270 // src1 * src2 - src3
14271 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14272   predicate(UseFMA);
14273   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14274 
14275   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14276 
14277   ins_encode %{
14278   // n.b. insn name should be fnmsubd
14279     __ fnmsub(as_FloatRegister($dst$$reg),
14280               as_FloatRegister($src1$$reg),
14281               as_FloatRegister($src2$$reg),
14282               as_FloatRegister($src3$$reg));
14283   %}
14284 
14285   ins_pipe(pipe_class_default);
14286 %}
14287 
14288 
14289 // Math.max(FF)F
14290 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14291   match(Set dst (MaxF src1 src2));
14292 
14293   format %{ "fmaxs   $dst, $src1, $src2" %}
14294   ins_encode %{
14295     __ fmaxs(as_FloatRegister($dst$$reg),
14296              as_FloatRegister($src1$$reg),
14297              as_FloatRegister($src2$$reg));
14298   %}
14299 
14300   ins_pipe(fp_dop_reg_reg_s);
14301 %}
14302 
14303 // Math.min(FF)F
14304 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14305   match(Set dst (MinF src1 src2));
14306 
14307   format %{ "fmins   $dst, $src1, $src2" %}
14308   ins_encode %{
14309     __ fmins(as_FloatRegister($dst$$reg),
14310              as_FloatRegister($src1$$reg),
14311              as_FloatRegister($src2$$reg));
14312   %}
14313 
14314   ins_pipe(fp_dop_reg_reg_s);
14315 %}
14316 
14317 // Math.max(DD)D
14318 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14319   match(Set dst (MaxD src1 src2));
14320 
14321   format %{ "fmaxd   $dst, $src1, $src2" %}
14322   ins_encode %{
14323     __ fmaxd(as_FloatRegister($dst$$reg),
14324              as_FloatRegister($src1$$reg),
14325              as_FloatRegister($src2$$reg));
14326   %}
14327 
14328   ins_pipe(fp_dop_reg_reg_d);
14329 %}
14330 
14331 // Math.min(DD)D
14332 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14333   match(Set dst (MinD src1 src2));
14334 
14335   format %{ "fmind   $dst, $src1, $src2" %}
14336   ins_encode %{
14337     __ fmind(as_FloatRegister($dst$$reg),
14338              as_FloatRegister($src1$$reg),
14339              as_FloatRegister($src2$$reg));
14340   %}
14341 
14342   ins_pipe(fp_dop_reg_reg_d);
14343 %}
14344 
14345 
14346 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14347   match(Set dst (DivF src1  src2));
14348 
14349   ins_cost(INSN_COST * 18);
14350   format %{ "fdivs   $dst, $src1, $src2" %}
14351 
14352   ins_encode %{
14353     __ fdivs(as_FloatRegister($dst$$reg),
14354              as_FloatRegister($src1$$reg),
14355              as_FloatRegister($src2$$reg));
14356   %}
14357 
14358   ins_pipe(fp_div_s);
14359 %}
14360 
14361 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14362   match(Set dst (DivD src1  src2));
14363 
14364   ins_cost(INSN_COST * 32);
14365   format %{ "fdivd   $dst, $src1, $src2" %}
14366 
14367   ins_encode %{
14368     __ fdivd(as_FloatRegister($dst$$reg),
14369              as_FloatRegister($src1$$reg),
14370              as_FloatRegister($src2$$reg));
14371   %}
14372 
14373   ins_pipe(fp_div_d);
14374 %}
14375 
14376 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14377   match(Set dst (NegF src));
14378 
14379   ins_cost(INSN_COST * 3);
14380   format %{ "fneg   $dst, $src" %}
14381 
14382   ins_encode %{
14383     __ fnegs(as_FloatRegister($dst$$reg),
14384              as_FloatRegister($src$$reg));
14385   %}
14386 
14387   ins_pipe(fp_uop_s);
14388 %}
14389 
14390 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14391   match(Set dst (NegD src));
14392 
14393   ins_cost(INSN_COST * 3);
14394   format %{ "fnegd   $dst, $src" %}
14395 
14396   ins_encode %{
14397     __ fnegd(as_FloatRegister($dst$$reg),
14398              as_FloatRegister($src$$reg));
14399   %}
14400 
14401   ins_pipe(fp_uop_d);
14402 %}
14403 
14404 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14405 %{
14406   match(Set dst (AbsI src));
14407 
14408   effect(KILL cr);
14409   ins_cost(INSN_COST * 2);
14410   format %{ "cmpw  $src, zr\n\t"
14411             "cnegw $dst, $src, Assembler::LT\t# int abs"
14412   %}
14413 
14414   ins_encode %{
14415     __ cmpw(as_Register($src$$reg), zr);
14416     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14417   %}
14418   ins_pipe(pipe_class_default);
14419 %}
14420 
14421 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14422 %{
14423   match(Set dst (AbsL src));
14424 
14425   effect(KILL cr);
14426   ins_cost(INSN_COST * 2);
14427   format %{ "cmp  $src, zr\n\t"
14428             "cneg $dst, $src, Assembler::LT\t# long abs"
14429   %}
14430 
14431   ins_encode %{
14432     __ cmp(as_Register($src$$reg), zr);
14433     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14434   %}
14435   ins_pipe(pipe_class_default);
14436 %}
14437 
14438 instruct absF_reg(vRegF dst, vRegF src) %{
14439   match(Set dst (AbsF src));
14440 
14441   ins_cost(INSN_COST * 3);
14442   format %{ "fabss   $dst, $src" %}
14443   ins_encode %{
14444     __ fabss(as_FloatRegister($dst$$reg),
14445              as_FloatRegister($src$$reg));
14446   %}
14447 
14448   ins_pipe(fp_uop_s);
14449 %}
14450 
14451 instruct absD_reg(vRegD dst, vRegD src) %{
14452   match(Set dst (AbsD src));
14453 
14454   ins_cost(INSN_COST * 3);
14455   format %{ "fabsd   $dst, $src" %}
14456   ins_encode %{
14457     __ fabsd(as_FloatRegister($dst$$reg),
14458              as_FloatRegister($src$$reg));
14459   %}
14460 
14461   ins_pipe(fp_uop_d);
14462 %}
14463 
14464 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14465   match(Set dst (AbsF (SubF src1 src2)));
14466 
14467   ins_cost(INSN_COST * 3);
14468   format %{ "fabds   $dst, $src1, $src2" %}
14469   ins_encode %{
14470     __ fabds(as_FloatRegister($dst$$reg),
14471              as_FloatRegister($src1$$reg),
14472              as_FloatRegister($src2$$reg));
14473   %}
14474 
14475   ins_pipe(fp_uop_s);
14476 %}
14477 
14478 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14479   match(Set dst (AbsD (SubD src1 src2)));
14480 
14481   ins_cost(INSN_COST * 3);
14482   format %{ "fabdd   $dst, $src1, $src2" %}
14483   ins_encode %{
14484     __ fabdd(as_FloatRegister($dst$$reg),
14485              as_FloatRegister($src1$$reg),
14486              as_FloatRegister($src2$$reg));
14487   %}
14488 
14489   ins_pipe(fp_uop_d);
14490 %}
14491 
14492 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14493   match(Set dst (SqrtD src));
14494 
14495   ins_cost(INSN_COST * 50);
14496   format %{ "fsqrtd  $dst, $src" %}
14497   ins_encode %{
14498     __ fsqrtd(as_FloatRegister($dst$$reg),
14499              as_FloatRegister($src$$reg));
14500   %}
14501 
14502   ins_pipe(fp_div_s);
14503 %}
14504 
14505 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14506   match(Set dst (SqrtF src));
14507 
14508   ins_cost(INSN_COST * 50);
14509   format %{ "fsqrts  $dst, $src" %}
14510   ins_encode %{
14511     __ fsqrts(as_FloatRegister($dst$$reg),
14512              as_FloatRegister($src$$reg));
14513   %}
14514 
14515   ins_pipe(fp_div_d);
14516 %}
14517 
14518 // Math.rint, floor, ceil
14519 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14520   match(Set dst (RoundDoubleMode src rmode));
14521   format %{ "frint  $dst, $src, $rmode" %}
14522   ins_encode %{
14523     switch ($rmode$$constant) {
14524       case RoundDoubleModeNode::rmode_rint:
14525         __ frintnd(as_FloatRegister($dst$$reg),
14526                    as_FloatRegister($src$$reg));
14527         break;
14528       case RoundDoubleModeNode::rmode_floor:
14529         __ frintmd(as_FloatRegister($dst$$reg),
14530                    as_FloatRegister($src$$reg));
14531         break;
14532       case RoundDoubleModeNode::rmode_ceil:
14533         __ frintpd(as_FloatRegister($dst$$reg),
14534                    as_FloatRegister($src$$reg));
14535         break;
14536     }
14537   %}
14538   ins_pipe(fp_uop_d);
14539 %}
14540 
14541 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14542   match(Set dst (CopySignD src1 (Binary src2 zero)));
14543   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14544   format %{ "CopySignD  $dst $src1 $src2" %}
14545   ins_encode %{
14546     FloatRegister dst = as_FloatRegister($dst$$reg),
14547                   src1 = as_FloatRegister($src1$$reg),
14548                   src2 = as_FloatRegister($src2$$reg),
14549                   zero = as_FloatRegister($zero$$reg);
14550     __ fnegd(dst, zero);
14551     __ bsl(dst, __ T8B, src2, src1);
14552   %}
14553   ins_pipe(fp_uop_d);
14554 %}
14555 
14556 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14557   match(Set dst (CopySignF src1 src2));
14558   effect(TEMP_DEF dst, USE src1, USE src2);
14559   format %{ "CopySignF  $dst $src1 $src2" %}
14560   ins_encode %{
14561     FloatRegister dst = as_FloatRegister($dst$$reg),
14562                   src1 = as_FloatRegister($src1$$reg),
14563                   src2 = as_FloatRegister($src2$$reg);
14564     __ movi(dst, __ T2S, 0x80, 24);
14565     __ bsl(dst, __ T8B, src2, src1);
14566   %}
14567   ins_pipe(fp_uop_d);
14568 %}
14569 
14570 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14571   match(Set dst (SignumD src (Binary zero one)));
14572   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14573   format %{ "signumD  $dst, $src" %}
14574   ins_encode %{
14575     FloatRegister src = as_FloatRegister($src$$reg),
14576                   dst = as_FloatRegister($dst$$reg),
14577                   zero = as_FloatRegister($zero$$reg),
14578                   one = as_FloatRegister($one$$reg);
14579     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14580     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14581     // Bit selection instruction gets bit from "one" for each enabled bit in
14582     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14583     // NaN the whole "src" will be copied because "dst" is zero. For all other
14584     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14585     // from "src", and all other bits are copied from 1.0.
14586     __ bsl(dst, __ T8B, one, src);
14587   %}
14588   ins_pipe(fp_uop_d);
14589 %}
14590 
14591 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14592   match(Set dst (SignumF src (Binary zero one)));
14593   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14594   format %{ "signumF  $dst, $src" %}
14595   ins_encode %{
14596     FloatRegister src = as_FloatRegister($src$$reg),
14597                   dst = as_FloatRegister($dst$$reg),
14598                   zero = as_FloatRegister($zero$$reg),
14599                   one = as_FloatRegister($one$$reg);
14600     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14601     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14602     // Bit selection instruction gets bit from "one" for each enabled bit in
14603     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14604     // NaN the whole "src" will be copied because "dst" is zero. For all other
14605     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14606     // from "src", and all other bits are copied from 1.0.
14607     __ bsl(dst, __ T8B, one, src);
14608   %}
14609   ins_pipe(fp_uop_d);
14610 %}
14611 
14612 instruct onspinwait() %{
14613   match(OnSpinWait);
14614   ins_cost(INSN_COST);
14615 
14616   format %{ "onspinwait" %}
14617 
14618   ins_encode %{
14619     __ spin_wait();
14620   %}
14621   ins_pipe(pipe_class_empty);
14622 %}
14623 
14624 // ============================================================================
14625 // Logical Instructions
14626 
14627 // Integer Logical Instructions
14628 
14629 // And Instructions
14630 
14631 
14632 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14633   match(Set dst (AndI src1 src2));
14634 
14635   format %{ "andw  $dst, $src1, $src2\t# int" %}
14636 
14637   ins_cost(INSN_COST);
14638   ins_encode %{
14639     __ andw(as_Register($dst$$reg),
14640             as_Register($src1$$reg),
14641             as_Register($src2$$reg));
14642   %}
14643 
14644   ins_pipe(ialu_reg_reg);
14645 %}
14646 
14647 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14648   match(Set dst (AndI src1 src2));
14649 
14650   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14651 
14652   ins_cost(INSN_COST);
14653   ins_encode %{
14654     __ andw(as_Register($dst$$reg),
14655             as_Register($src1$$reg),
14656             (uint64_t)($src2$$constant));
14657   %}
14658 
14659   ins_pipe(ialu_reg_imm);
14660 %}
14661 
14662 // Or Instructions
14663 
14664 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14665   match(Set dst (OrI src1 src2));
14666 
14667   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14668 
14669   ins_cost(INSN_COST);
14670   ins_encode %{
14671     __ orrw(as_Register($dst$$reg),
14672             as_Register($src1$$reg),
14673             as_Register($src2$$reg));
14674   %}
14675 
14676   ins_pipe(ialu_reg_reg);
14677 %}
14678 
14679 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14680   match(Set dst (OrI src1 src2));
14681 
14682   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14683 
14684   ins_cost(INSN_COST);
14685   ins_encode %{
14686     __ orrw(as_Register($dst$$reg),
14687             as_Register($src1$$reg),
14688             (uint64_t)($src2$$constant));
14689   %}
14690 
14691   ins_pipe(ialu_reg_imm);
14692 %}
14693 
14694 // Xor Instructions
14695 
14696 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14697   match(Set dst (XorI src1 src2));
14698 
14699   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14700 
14701   ins_cost(INSN_COST);
14702   ins_encode %{
14703     __ eorw(as_Register($dst$$reg),
14704             as_Register($src1$$reg),
14705             as_Register($src2$$reg));
14706   %}
14707 
14708   ins_pipe(ialu_reg_reg);
14709 %}
14710 
14711 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14712   match(Set dst (XorI src1 src2));
14713 
14714   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14715 
14716   ins_cost(INSN_COST);
14717   ins_encode %{
14718     __ eorw(as_Register($dst$$reg),
14719             as_Register($src1$$reg),
14720             (uint64_t)($src2$$constant));
14721   %}
14722 
14723   ins_pipe(ialu_reg_imm);
14724 %}
14725 
14726 // Long Logical Instructions
14727 // TODO
14728 
14729 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14730   match(Set dst (AndL src1 src2));
14731 
14732   format %{ "and  $dst, $src1, $src2\t# int" %}
14733 
14734   ins_cost(INSN_COST);
14735   ins_encode %{
14736     __ andr(as_Register($dst$$reg),
14737             as_Register($src1$$reg),
14738             as_Register($src2$$reg));
14739   %}
14740 
14741   ins_pipe(ialu_reg_reg);
14742 %}
14743 
14744 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14745   match(Set dst (AndL src1 src2));
14746 
14747   format %{ "and  $dst, $src1, $src2\t# int" %}
14748 
14749   ins_cost(INSN_COST);
14750   ins_encode %{
14751     __ andr(as_Register($dst$$reg),
14752             as_Register($src1$$reg),
14753             (uint64_t)($src2$$constant));
14754   %}
14755 
14756   ins_pipe(ialu_reg_imm);
14757 %}
14758 
14759 // Or Instructions
14760 
14761 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14762   match(Set dst (OrL src1 src2));
14763 
14764   format %{ "orr  $dst, $src1, $src2\t# int" %}
14765 
14766   ins_cost(INSN_COST);
14767   ins_encode %{
14768     __ orr(as_Register($dst$$reg),
14769            as_Register($src1$$reg),
14770            as_Register($src2$$reg));
14771   %}
14772 
14773   ins_pipe(ialu_reg_reg);
14774 %}
14775 
14776 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14777   match(Set dst (OrL src1 src2));
14778 
14779   format %{ "orr  $dst, $src1, $src2\t# int" %}
14780 
14781   ins_cost(INSN_COST);
14782   ins_encode %{
14783     __ orr(as_Register($dst$$reg),
14784            as_Register($src1$$reg),
14785            (uint64_t)($src2$$constant));
14786   %}
14787 
14788   ins_pipe(ialu_reg_imm);
14789 %}
14790 
14791 // Xor Instructions
14792 
14793 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14794   match(Set dst (XorL src1 src2));
14795 
14796   format %{ "eor  $dst, $src1, $src2\t# int" %}
14797 
14798   ins_cost(INSN_COST);
14799   ins_encode %{
14800     __ eor(as_Register($dst$$reg),
14801            as_Register($src1$$reg),
14802            as_Register($src2$$reg));
14803   %}
14804 
14805   ins_pipe(ialu_reg_reg);
14806 %}
14807 
14808 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14809   match(Set dst (XorL src1 src2));
14810 
14811   ins_cost(INSN_COST);
14812   format %{ "eor  $dst, $src1, $src2\t# int" %}
14813 
14814   ins_encode %{
14815     __ eor(as_Register($dst$$reg),
14816            as_Register($src1$$reg),
14817            (uint64_t)($src2$$constant));
14818   %}
14819 
14820   ins_pipe(ialu_reg_imm);
14821 %}
14822 
14823 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14824 %{
14825   match(Set dst (ConvI2L src));
14826 
14827   ins_cost(INSN_COST);
14828   format %{ "sxtw  $dst, $src\t# i2l" %}
14829   ins_encode %{
14830     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14831   %}
14832   ins_pipe(ialu_reg_shift);
14833 %}
14834 
14835 // this pattern occurs in bigmath arithmetic
14836 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14837 %{
14838   match(Set dst (AndL (ConvI2L src) mask));
14839 
14840   ins_cost(INSN_COST);
14841   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14842   ins_encode %{
14843     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14844   %}
14845 
14846   ins_pipe(ialu_reg_shift);
14847 %}
14848 
14849 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14850   match(Set dst (ConvL2I src));
14851 
14852   ins_cost(INSN_COST);
14853   format %{ "movw  $dst, $src \t// l2i" %}
14854 
14855   ins_encode %{
14856     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14857   %}
14858 
14859   ins_pipe(ialu_reg);
14860 %}
14861 
14862 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14863 %{
14864   match(Set dst (Conv2B src));
14865   effect(KILL cr);
14866 
14867   format %{
14868     "cmpw $src, zr\n\t"
14869     "cset $dst, ne"
14870   %}
14871 
14872   ins_encode %{
14873     __ cmpw(as_Register($src$$reg), zr);
14874     __ cset(as_Register($dst$$reg), Assembler::NE);
14875   %}
14876 
14877   ins_pipe(ialu_reg);
14878 %}
14879 
14880 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14881 %{
14882   match(Set dst (Conv2B src));
14883   effect(KILL cr);
14884 
14885   format %{
14886     "cmp  $src, zr\n\t"
14887     "cset $dst, ne"
14888   %}
14889 
14890   ins_encode %{
14891     __ cmp(as_Register($src$$reg), zr);
14892     __ cset(as_Register($dst$$reg), Assembler::NE);
14893   %}
14894 
14895   ins_pipe(ialu_reg);
14896 %}
14897 
14898 instruct convD2F_reg(vRegF dst, vRegD src) %{
14899   match(Set dst (ConvD2F src));
14900 
14901   ins_cost(INSN_COST * 5);
14902   format %{ "fcvtd  $dst, $src \t// d2f" %}
14903 
14904   ins_encode %{
14905     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14906   %}
14907 
14908   ins_pipe(fp_d2f);
14909 %}
14910 
14911 instruct convF2D_reg(vRegD dst, vRegF src) %{
14912   match(Set dst (ConvF2D src));
14913 
14914   ins_cost(INSN_COST * 5);
14915   format %{ "fcvts  $dst, $src \t// f2d" %}
14916 
14917   ins_encode %{
14918     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14919   %}
14920 
14921   ins_pipe(fp_f2d);
14922 %}
14923 
14924 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14925   match(Set dst (ConvF2I src));
14926 
14927   ins_cost(INSN_COST * 5);
14928   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14929 
14930   ins_encode %{
14931     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14932   %}
14933 
14934   ins_pipe(fp_f2i);
14935 %}
14936 
14937 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14938   match(Set dst (ConvF2L src));
14939 
14940   ins_cost(INSN_COST * 5);
14941   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14942 
14943   ins_encode %{
14944     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14945   %}
14946 
14947   ins_pipe(fp_f2l);
14948 %}
14949 
14950 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14951   match(Set dst (ConvI2F src));
14952 
14953   ins_cost(INSN_COST * 5);
14954   format %{ "scvtfws  $dst, $src \t// i2f" %}
14955 
14956   ins_encode %{
14957     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14958   %}
14959 
14960   ins_pipe(fp_i2f);
14961 %}
14962 
14963 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14964   match(Set dst (ConvL2F src));
14965 
14966   ins_cost(INSN_COST * 5);
14967   format %{ "scvtfs  $dst, $src \t// l2f" %}
14968 
14969   ins_encode %{
14970     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14971   %}
14972 
14973   ins_pipe(fp_l2f);
14974 %}
14975 
14976 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14977   match(Set dst (ConvD2I src));
14978 
14979   ins_cost(INSN_COST * 5);
14980   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14981 
14982   ins_encode %{
14983     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14984   %}
14985 
14986   ins_pipe(fp_d2i);
14987 %}
14988 
14989 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14990   match(Set dst (ConvD2L src));
14991 
14992   ins_cost(INSN_COST * 5);
14993   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14994 
14995   ins_encode %{
14996     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14997   %}
14998 
14999   ins_pipe(fp_d2l);
15000 %}
15001 
15002 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15003   match(Set dst (ConvI2D src));
15004 
15005   ins_cost(INSN_COST * 5);
15006   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15007 
15008   ins_encode %{
15009     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15010   %}
15011 
15012   ins_pipe(fp_i2d);
15013 %}
15014 
15015 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15016   match(Set dst (ConvL2D src));
15017 
15018   ins_cost(INSN_COST * 5);
15019   format %{ "scvtfd  $dst, $src \t// l2d" %}
15020 
15021   ins_encode %{
15022     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15023   %}
15024 
15025   ins_pipe(fp_l2d);
15026 %}
15027 
15028 // stack <-> reg and reg <-> reg shuffles with no conversion
15029 
15030 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15031 
15032   match(Set dst (MoveF2I src));
15033 
15034   effect(DEF dst, USE src);
15035 
15036   ins_cost(4 * INSN_COST);
15037 
15038   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15039 
15040   ins_encode %{
15041     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15042   %}
15043 
15044   ins_pipe(iload_reg_reg);
15045 
15046 %}
15047 
15048 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15049 
15050   match(Set dst (MoveI2F src));
15051 
15052   effect(DEF dst, USE src);
15053 
15054   ins_cost(4 * INSN_COST);
15055 
15056   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15057 
15058   ins_encode %{
15059     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15060   %}
15061 
15062   ins_pipe(pipe_class_memory);
15063 
15064 %}
15065 
15066 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15067 
15068   match(Set dst (MoveD2L src));
15069 
15070   effect(DEF dst, USE src);
15071 
15072   ins_cost(4 * INSN_COST);
15073 
15074   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15075 
15076   ins_encode %{
15077     __ ldr($dst$$Register, Address(sp, $src$$disp));
15078   %}
15079 
15080   ins_pipe(iload_reg_reg);
15081 
15082 %}
15083 
15084 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15085 
15086   match(Set dst (MoveL2D src));
15087 
15088   effect(DEF dst, USE src);
15089 
15090   ins_cost(4 * INSN_COST);
15091 
15092   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15093 
15094   ins_encode %{
15095     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15096   %}
15097 
15098   ins_pipe(pipe_class_memory);
15099 
15100 %}
15101 
15102 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15103 
15104   match(Set dst (MoveF2I src));
15105 
15106   effect(DEF dst, USE src);
15107 
15108   ins_cost(INSN_COST);
15109 
15110   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15111 
15112   ins_encode %{
15113     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15114   %}
15115 
15116   ins_pipe(pipe_class_memory);
15117 
15118 %}
15119 
15120 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15121 
15122   match(Set dst (MoveI2F src));
15123 
15124   effect(DEF dst, USE src);
15125 
15126   ins_cost(INSN_COST);
15127 
15128   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15129 
15130   ins_encode %{
15131     __ strw($src$$Register, Address(sp, $dst$$disp));
15132   %}
15133 
15134   ins_pipe(istore_reg_reg);
15135 
15136 %}
15137 
15138 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15139 
15140   match(Set dst (MoveD2L src));
15141 
15142   effect(DEF dst, USE src);
15143 
15144   ins_cost(INSN_COST);
15145 
15146   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15147 
15148   ins_encode %{
15149     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15150   %}
15151 
15152   ins_pipe(pipe_class_memory);
15153 
15154 %}
15155 
15156 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15157 
15158   match(Set dst (MoveL2D src));
15159 
15160   effect(DEF dst, USE src);
15161 
15162   ins_cost(INSN_COST);
15163 
15164   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15165 
15166   ins_encode %{
15167     __ str($src$$Register, Address(sp, $dst$$disp));
15168   %}
15169 
15170   ins_pipe(istore_reg_reg);
15171 
15172 %}
15173 
15174 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15175 
15176   match(Set dst (MoveF2I src));
15177 
15178   effect(DEF dst, USE src);
15179 
15180   ins_cost(INSN_COST);
15181 
15182   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15183 
15184   ins_encode %{
15185     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15186   %}
15187 
15188   ins_pipe(fp_f2i);
15189 
15190 %}
15191 
15192 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15193 
15194   match(Set dst (MoveI2F src));
15195 
15196   effect(DEF dst, USE src);
15197 
15198   ins_cost(INSN_COST);
15199 
15200   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15201 
15202   ins_encode %{
15203     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15204   %}
15205 
15206   ins_pipe(fp_i2f);
15207 
15208 %}
15209 
15210 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15211 
15212   match(Set dst (MoveD2L src));
15213 
15214   effect(DEF dst, USE src);
15215 
15216   ins_cost(INSN_COST);
15217 
15218   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15219 
15220   ins_encode %{
15221     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15222   %}
15223 
15224   ins_pipe(fp_d2l);
15225 
15226 %}
15227 
15228 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15229 
15230   match(Set dst (MoveL2D src));
15231 
15232   effect(DEF dst, USE src);
15233 
15234   ins_cost(INSN_COST);
15235 
15236   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15237 
15238   ins_encode %{
15239     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15240   %}
15241 
15242   ins_pipe(fp_l2d);
15243 
15244 %}
15245 
15246 // ============================================================================
15247 // clearing of an array
15248 
15249 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15250 %{
15251   match(Set dummy (ClearArray cnt base));
15252   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15253 
15254   ins_cost(4 * INSN_COST);
15255   format %{ "ClearArray $cnt, $base" %}
15256 
15257   ins_encode %{
15258     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15259     if (tpc == NULL) {
15260       ciEnv::current()->record_failure("CodeCache is full");
15261       return;
15262     }
15263   %}
15264 
15265   ins_pipe(pipe_class_memory);
15266 %}
15267 
15268 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15269 %{
15270   predicate((uint64_t)n->in(2)->get_long()
15271             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15272   match(Set dummy (ClearArray cnt base));
15273   effect(TEMP temp, USE_KILL base, KILL cr);
15274 
15275   ins_cost(4 * INSN_COST);
15276   format %{ "ClearArray $cnt, $base" %}
15277 
15278   ins_encode %{
15279     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15280     if (tpc == NULL) {
15281       ciEnv::current()->record_failure("CodeCache is full");
15282       return;
15283     }
15284   %}
15285 
15286   ins_pipe(pipe_class_memory);
15287 %}
15288 
15289 // ============================================================================
15290 // Overflow Math Instructions
15291 
15292 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15293 %{
15294   match(Set cr (OverflowAddI op1 op2));
15295 
15296   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15297   ins_cost(INSN_COST);
15298   ins_encode %{
15299     __ cmnw($op1$$Register, $op2$$Register);
15300   %}
15301 
15302   ins_pipe(icmp_reg_reg);
15303 %}
15304 
15305 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15306 %{
15307   match(Set cr (OverflowAddI op1 op2));
15308 
15309   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15310   ins_cost(INSN_COST);
15311   ins_encode %{
15312     __ cmnw($op1$$Register, $op2$$constant);
15313   %}
15314 
15315   ins_pipe(icmp_reg_imm);
15316 %}
15317 
15318 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15319 %{
15320   match(Set cr (OverflowAddL op1 op2));
15321 
15322   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15323   ins_cost(INSN_COST);
15324   ins_encode %{
15325     __ cmn($op1$$Register, $op2$$Register);
15326   %}
15327 
15328   ins_pipe(icmp_reg_reg);
15329 %}
15330 
15331 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15332 %{
15333   match(Set cr (OverflowAddL op1 op2));
15334 
15335   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15336   ins_cost(INSN_COST);
15337   ins_encode %{
15338     __ adds(zr, $op1$$Register, $op2$$constant);
15339   %}
15340 
15341   ins_pipe(icmp_reg_imm);
15342 %}
15343 
15344 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15345 %{
15346   match(Set cr (OverflowSubI op1 op2));
15347 
15348   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15349   ins_cost(INSN_COST);
15350   ins_encode %{
15351     __ cmpw($op1$$Register, $op2$$Register);
15352   %}
15353 
15354   ins_pipe(icmp_reg_reg);
15355 %}
15356 
15357 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15358 %{
15359   match(Set cr (OverflowSubI op1 op2));
15360 
15361   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15362   ins_cost(INSN_COST);
15363   ins_encode %{
15364     __ cmpw($op1$$Register, $op2$$constant);
15365   %}
15366 
15367   ins_pipe(icmp_reg_imm);
15368 %}
15369 
15370 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15371 %{
15372   match(Set cr (OverflowSubL op1 op2));
15373 
15374   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15375   ins_cost(INSN_COST);
15376   ins_encode %{
15377     __ cmp($op1$$Register, $op2$$Register);
15378   %}
15379 
15380   ins_pipe(icmp_reg_reg);
15381 %}
15382 
15383 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15384 %{
15385   match(Set cr (OverflowSubL op1 op2));
15386 
15387   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15388   ins_cost(INSN_COST);
15389   ins_encode %{
15390     __ subs(zr, $op1$$Register, $op2$$constant);
15391   %}
15392 
15393   ins_pipe(icmp_reg_imm);
15394 %}
15395 
15396 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15397 %{
15398   match(Set cr (OverflowSubI zero op1));
15399 
15400   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15401   ins_cost(INSN_COST);
15402   ins_encode %{
15403     __ cmpw(zr, $op1$$Register);
15404   %}
15405 
15406   ins_pipe(icmp_reg_imm);
15407 %}
15408 
15409 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15410 %{
15411   match(Set cr (OverflowSubL zero op1));
15412 
15413   format %{ "cmp   zr, $op1\t# overflow check long" %}
15414   ins_cost(INSN_COST);
15415   ins_encode %{
15416     __ cmp(zr, $op1$$Register);
15417   %}
15418 
15419   ins_pipe(icmp_reg_imm);
15420 %}
15421 
15422 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15423 %{
15424   match(Set cr (OverflowMulI op1 op2));
15425 
15426   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15427             "cmp   rscratch1, rscratch1, sxtw\n\t"
15428             "movw  rscratch1, #0x80000000\n\t"
15429             "cselw rscratch1, rscratch1, zr, NE\n\t"
15430             "cmpw  rscratch1, #1" %}
15431   ins_cost(5 * INSN_COST);
15432   ins_encode %{
15433     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15434     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15435     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15436     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15437     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15438   %}
15439 
15440   ins_pipe(pipe_slow);
15441 %}
15442 
15443 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15444 %{
15445   match(If cmp (OverflowMulI op1 op2));
15446   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15447             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15448   effect(USE labl, KILL cr);
15449 
15450   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15451             "cmp   rscratch1, rscratch1, sxtw\n\t"
15452             "b$cmp   $labl" %}
15453   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15454   ins_encode %{
15455     Label* L = $labl$$label;
15456     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15457     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15458     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15459     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15460   %}
15461 
15462   ins_pipe(pipe_serial);
15463 %}
15464 
15465 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15466 %{
15467   match(Set cr (OverflowMulL op1 op2));
15468 
15469   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15470             "smulh rscratch2, $op1, $op2\n\t"
15471             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15472             "movw  rscratch1, #0x80000000\n\t"
15473             "cselw rscratch1, rscratch1, zr, NE\n\t"
15474             "cmpw  rscratch1, #1" %}
15475   ins_cost(6 * INSN_COST);
15476   ins_encode %{
15477     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15478     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15479     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15480     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15481     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15482     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15483   %}
15484 
15485   ins_pipe(pipe_slow);
15486 %}
15487 
15488 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15489 %{
15490   match(If cmp (OverflowMulL op1 op2));
15491   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15492             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15493   effect(USE labl, KILL cr);
15494 
15495   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15496             "smulh rscratch2, $op1, $op2\n\t"
15497             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15498             "b$cmp $labl" %}
15499   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15500   ins_encode %{
15501     Label* L = $labl$$label;
15502     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15503     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15504     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15505     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15506     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15507   %}
15508 
15509   ins_pipe(pipe_serial);
15510 %}
15511 
15512 // ============================================================================
15513 // Compare Instructions
15514 
15515 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15516 %{
15517   match(Set cr (CmpI op1 op2));
15518 
15519   effect(DEF cr, USE op1, USE op2);
15520 
15521   ins_cost(INSN_COST);
15522   format %{ "cmpw  $op1, $op2" %}
15523 
15524   ins_encode(aarch64_enc_cmpw(op1, op2));
15525 
15526   ins_pipe(icmp_reg_reg);
15527 %}
15528 
15529 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15530 %{
15531   match(Set cr (CmpI op1 zero));
15532 
15533   effect(DEF cr, USE op1);
15534 
15535   ins_cost(INSN_COST);
15536   format %{ "cmpw $op1, 0" %}
15537 
15538   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15539 
15540   ins_pipe(icmp_reg_imm);
15541 %}
15542 
15543 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15544 %{
15545   match(Set cr (CmpI op1 op2));
15546 
15547   effect(DEF cr, USE op1);
15548 
15549   ins_cost(INSN_COST);
15550   format %{ "cmpw  $op1, $op2" %}
15551 
15552   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15553 
15554   ins_pipe(icmp_reg_imm);
15555 %}
15556 
15557 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15558 %{
15559   match(Set cr (CmpI op1 op2));
15560 
15561   effect(DEF cr, USE op1);
15562 
15563   ins_cost(INSN_COST * 2);
15564   format %{ "cmpw  $op1, $op2" %}
15565 
15566   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15567 
15568   ins_pipe(icmp_reg_imm);
15569 %}
15570 
15571 // Unsigned compare Instructions; really, same as signed compare
15572 // except it should only be used to feed an If or a CMovI which takes a
15573 // cmpOpU.
15574 
15575 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15576 %{
15577   match(Set cr (CmpU op1 op2));
15578 
15579   effect(DEF cr, USE op1, USE op2);
15580 
15581   ins_cost(INSN_COST);
15582   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15583 
15584   ins_encode(aarch64_enc_cmpw(op1, op2));
15585 
15586   ins_pipe(icmp_reg_reg);
15587 %}
15588 
15589 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15590 %{
15591   match(Set cr (CmpU op1 zero));
15592 
15593   effect(DEF cr, USE op1);
15594 
15595   ins_cost(INSN_COST);
15596   format %{ "cmpw $op1, #0\t# unsigned" %}
15597 
15598   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15599 
15600   ins_pipe(icmp_reg_imm);
15601 %}
15602 
15603 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15604 %{
15605   match(Set cr (CmpU op1 op2));
15606 
15607   effect(DEF cr, USE op1);
15608 
15609   ins_cost(INSN_COST);
15610   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15611 
15612   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15613 
15614   ins_pipe(icmp_reg_imm);
15615 %}
15616 
15617 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15618 %{
15619   match(Set cr (CmpU op1 op2));
15620 
15621   effect(DEF cr, USE op1);
15622 
15623   ins_cost(INSN_COST * 2);
15624   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15625 
15626   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15627 
15628   ins_pipe(icmp_reg_imm);
15629 %}
15630 
15631 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15632 %{
15633   match(Set cr (CmpL op1 op2));
15634 
15635   effect(DEF cr, USE op1, USE op2);
15636 
15637   ins_cost(INSN_COST);
15638   format %{ "cmp  $op1, $op2" %}
15639 
15640   ins_encode(aarch64_enc_cmp(op1, op2));
15641 
15642   ins_pipe(icmp_reg_reg);
15643 %}
15644 
15645 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15646 %{
15647   match(Set cr (CmpL op1 zero));
15648 
15649   effect(DEF cr, USE op1);
15650 
15651   ins_cost(INSN_COST);
15652   format %{ "tst  $op1" %}
15653 
15654   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15655 
15656   ins_pipe(icmp_reg_imm);
15657 %}
15658 
15659 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15660 %{
15661   match(Set cr (CmpL op1 op2));
15662 
15663   effect(DEF cr, USE op1);
15664 
15665   ins_cost(INSN_COST);
15666   format %{ "cmp  $op1, $op2" %}
15667 
15668   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15669 
15670   ins_pipe(icmp_reg_imm);
15671 %}
15672 
15673 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15674 %{
15675   match(Set cr (CmpL op1 op2));
15676 
15677   effect(DEF cr, USE op1);
15678 
15679   ins_cost(INSN_COST * 2);
15680   format %{ "cmp  $op1, $op2" %}
15681 
15682   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15683 
15684   ins_pipe(icmp_reg_imm);
15685 %}
15686 
15687 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15688 %{
15689   match(Set cr (CmpUL op1 op2));
15690 
15691   effect(DEF cr, USE op1, USE op2);
15692 
15693   ins_cost(INSN_COST);
15694   format %{ "cmp  $op1, $op2" %}
15695 
15696   ins_encode(aarch64_enc_cmp(op1, op2));
15697 
15698   ins_pipe(icmp_reg_reg);
15699 %}
15700 
15701 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15702 %{
15703   match(Set cr (CmpUL op1 zero));
15704 
15705   effect(DEF cr, USE op1);
15706 
15707   ins_cost(INSN_COST);
15708   format %{ "tst  $op1" %}
15709 
15710   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15711 
15712   ins_pipe(icmp_reg_imm);
15713 %}
15714 
15715 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15716 %{
15717   match(Set cr (CmpUL op1 op2));
15718 
15719   effect(DEF cr, USE op1);
15720 
15721   ins_cost(INSN_COST);
15722   format %{ "cmp  $op1, $op2" %}
15723 
15724   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15725 
15726   ins_pipe(icmp_reg_imm);
15727 %}
15728 
15729 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15730 %{
15731   match(Set cr (CmpUL op1 op2));
15732 
15733   effect(DEF cr, USE op1);
15734 
15735   ins_cost(INSN_COST * 2);
15736   format %{ "cmp  $op1, $op2" %}
15737 
15738   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15739 
15740   ins_pipe(icmp_reg_imm);
15741 %}
15742 
15743 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15744 %{
15745   match(Set cr (CmpP op1 op2));
15746 
15747   effect(DEF cr, USE op1, USE op2);
15748 
15749   ins_cost(INSN_COST);
15750   format %{ "cmp  $op1, $op2\t // ptr" %}
15751 
15752   ins_encode(aarch64_enc_cmpp(op1, op2));
15753 
15754   ins_pipe(icmp_reg_reg);
15755 %}
15756 
15757 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15758 %{
15759   match(Set cr (CmpN op1 op2));
15760 
15761   effect(DEF cr, USE op1, USE op2);
15762 
15763   ins_cost(INSN_COST);
15764   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15765 
15766   ins_encode(aarch64_enc_cmpn(op1, op2));
15767 
15768   ins_pipe(icmp_reg_reg);
15769 %}
15770 
15771 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15772 %{
15773   match(Set cr (CmpP op1 zero));
15774 
15775   effect(DEF cr, USE op1, USE zero);
15776 
15777   ins_cost(INSN_COST);
15778   format %{ "cmp  $op1, 0\t // ptr" %}
15779 
15780   ins_encode(aarch64_enc_testp(op1));
15781 
15782   ins_pipe(icmp_reg_imm);
15783 %}
15784 
15785 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15786 %{
15787   match(Set cr (CmpN op1 zero));
15788 
15789   effect(DEF cr, USE op1, USE zero);
15790 
15791   ins_cost(INSN_COST);
15792   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15793 
15794   ins_encode(aarch64_enc_testn(op1));
15795 
15796   ins_pipe(icmp_reg_imm);
15797 %}
15798 
15799 // FP comparisons
15800 //
15801 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15802 // using normal cmpOp. See declaration of rFlagsReg for details.
15803 
15804 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15805 %{
15806   match(Set cr (CmpF src1 src2));
15807 
15808   ins_cost(3 * INSN_COST);
15809   format %{ "fcmps $src1, $src2" %}
15810 
15811   ins_encode %{
15812     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15813   %}
15814 
15815   ins_pipe(pipe_class_compare);
15816 %}
15817 
15818 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15819 %{
15820   match(Set cr (CmpF src1 src2));
15821 
15822   ins_cost(3 * INSN_COST);
15823   format %{ "fcmps $src1, 0.0" %}
15824 
15825   ins_encode %{
15826     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15827   %}
15828 
15829   ins_pipe(pipe_class_compare);
15830 %}
15831 // FROM HERE
15832 
15833 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15834 %{
15835   match(Set cr (CmpD src1 src2));
15836 
15837   ins_cost(3 * INSN_COST);
15838   format %{ "fcmpd $src1, $src2" %}
15839 
15840   ins_encode %{
15841     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15842   %}
15843 
15844   ins_pipe(pipe_class_compare);
15845 %}
15846 
15847 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15848 %{
15849   match(Set cr (CmpD src1 src2));
15850 
15851   ins_cost(3 * INSN_COST);
15852   format %{ "fcmpd $src1, 0.0" %}
15853 
15854   ins_encode %{
15855     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15856   %}
15857 
15858   ins_pipe(pipe_class_compare);
15859 %}
15860 
15861 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15862 %{
15863   match(Set dst (CmpF3 src1 src2));
15864   effect(KILL cr);
15865 
15866   ins_cost(5 * INSN_COST);
15867   format %{ "fcmps $src1, $src2\n\t"
15868             "csinvw($dst, zr, zr, eq\n\t"
15869             "csnegw($dst, $dst, $dst, lt)"
15870   %}
15871 
15872   ins_encode %{
15873     Label done;
15874     FloatRegister s1 = as_FloatRegister($src1$$reg);
15875     FloatRegister s2 = as_FloatRegister($src2$$reg);
15876     Register d = as_Register($dst$$reg);
15877     __ fcmps(s1, s2);
15878     // installs 0 if EQ else -1
15879     __ csinvw(d, zr, zr, Assembler::EQ);
15880     // keeps -1 if less or unordered else installs 1
15881     __ csnegw(d, d, d, Assembler::LT);
15882     __ bind(done);
15883   %}
15884 
15885   ins_pipe(pipe_class_default);
15886 
15887 %}
15888 
15889 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15890 %{
15891   match(Set dst (CmpD3 src1 src2));
15892   effect(KILL cr);
15893 
15894   ins_cost(5 * INSN_COST);
15895   format %{ "fcmpd $src1, $src2\n\t"
15896             "csinvw($dst, zr, zr, eq\n\t"
15897             "csnegw($dst, $dst, $dst, lt)"
15898   %}
15899 
15900   ins_encode %{
15901     Label done;
15902     FloatRegister s1 = as_FloatRegister($src1$$reg);
15903     FloatRegister s2 = as_FloatRegister($src2$$reg);
15904     Register d = as_Register($dst$$reg);
15905     __ fcmpd(s1, s2);
15906     // installs 0 if EQ else -1
15907     __ csinvw(d, zr, zr, Assembler::EQ);
15908     // keeps -1 if less or unordered else installs 1
15909     __ csnegw(d, d, d, Assembler::LT);
15910     __ bind(done);
15911   %}
15912   ins_pipe(pipe_class_default);
15913 
15914 %}
15915 
15916 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15917 %{
15918   match(Set dst (CmpF3 src1 zero));
15919   effect(KILL cr);
15920 
15921   ins_cost(5 * INSN_COST);
15922   format %{ "fcmps $src1, 0.0\n\t"
15923             "csinvw($dst, zr, zr, eq\n\t"
15924             "csnegw($dst, $dst, $dst, lt)"
15925   %}
15926 
15927   ins_encode %{
15928     Label done;
15929     FloatRegister s1 = as_FloatRegister($src1$$reg);
15930     Register d = as_Register($dst$$reg);
15931     __ fcmps(s1, 0.0);
15932     // installs 0 if EQ else -1
15933     __ csinvw(d, zr, zr, Assembler::EQ);
15934     // keeps -1 if less or unordered else installs 1
15935     __ csnegw(d, d, d, Assembler::LT);
15936     __ bind(done);
15937   %}
15938 
15939   ins_pipe(pipe_class_default);
15940 
15941 %}
15942 
15943 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15944 %{
15945   match(Set dst (CmpD3 src1 zero));
15946   effect(KILL cr);
15947 
15948   ins_cost(5 * INSN_COST);
15949   format %{ "fcmpd $src1, 0.0\n\t"
15950             "csinvw($dst, zr, zr, eq\n\t"
15951             "csnegw($dst, $dst, $dst, lt)"
15952   %}
15953 
15954   ins_encode %{
15955     Label done;
15956     FloatRegister s1 = as_FloatRegister($src1$$reg);
15957     Register d = as_Register($dst$$reg);
15958     __ fcmpd(s1, 0.0);
15959     // installs 0 if EQ else -1
15960     __ csinvw(d, zr, zr, Assembler::EQ);
15961     // keeps -1 if less or unordered else installs 1
15962     __ csnegw(d, d, d, Assembler::LT);
15963     __ bind(done);
15964   %}
15965   ins_pipe(pipe_class_default);
15966 
15967 %}
15968 
15969 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15970 %{
15971   match(Set dst (CmpLTMask p q));
15972   effect(KILL cr);
15973 
15974   ins_cost(3 * INSN_COST);
15975 
15976   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15977             "csetw $dst, lt\n\t"
15978             "subw $dst, zr, $dst"
15979   %}
15980 
15981   ins_encode %{
15982     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15983     __ csetw(as_Register($dst$$reg), Assembler::LT);
15984     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15985   %}
15986 
15987   ins_pipe(ialu_reg_reg);
15988 %}
15989 
15990 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15991 %{
15992   match(Set dst (CmpLTMask src zero));
15993   effect(KILL cr);
15994 
15995   ins_cost(INSN_COST);
15996 
15997   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15998 
15999   ins_encode %{
16000     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16001   %}
16002 
16003   ins_pipe(ialu_reg_shift);
16004 %}
16005 
16006 // ============================================================================
16007 // Max and Min
16008 
16009 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16010 %{
16011   effect( DEF dst, USE src1, USE src2, USE cr );
16012 
16013   ins_cost(INSN_COST * 2);
16014   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
16015 
16016   ins_encode %{
16017     __ cselw(as_Register($dst$$reg),
16018              as_Register($src1$$reg),
16019              as_Register($src2$$reg),
16020              Assembler::LT);
16021   %}
16022 
16023   ins_pipe(icond_reg_reg);
16024 %}
16025 
16026 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16027 %{
16028   match(Set dst (MinI src1 src2));
16029   ins_cost(INSN_COST * 3);
16030 
16031   expand %{
16032     rFlagsReg cr;
16033     compI_reg_reg(cr, src1, src2);
16034     cmovI_reg_reg_lt(dst, src1, src2, cr);
16035   %}
16036 
16037 %}
16038 // FROM HERE
16039 
16040 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16041 %{
16042   effect( DEF dst, USE src1, USE src2, USE cr );
16043 
16044   ins_cost(INSN_COST * 2);
16045   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
16046 
16047   ins_encode %{
16048     __ cselw(as_Register($dst$$reg),
16049              as_Register($src1$$reg),
16050              as_Register($src2$$reg),
16051              Assembler::GT);
16052   %}
16053 
16054   ins_pipe(icond_reg_reg);
16055 %}
16056 
16057 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16058 %{
16059   match(Set dst (MaxI src1 src2));
16060   ins_cost(INSN_COST * 3);
16061   expand %{
16062     rFlagsReg cr;
16063     compI_reg_reg(cr, src1, src2);
16064     cmovI_reg_reg_gt(dst, src1, src2, cr);
16065   %}
16066 %}
16067 
16068 // ============================================================================
16069 // Branch Instructions
16070 
16071 // Direct Branch.
16072 instruct branch(label lbl)
16073 %{
16074   match(Goto);
16075 
16076   effect(USE lbl);
16077 
16078   ins_cost(BRANCH_COST);
16079   format %{ "b  $lbl" %}
16080 
16081   ins_encode(aarch64_enc_b(lbl));
16082 
16083   ins_pipe(pipe_branch);
16084 %}
16085 
16086 // Conditional Near Branch
16087 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16088 %{
16089   // Same match rule as `branchConFar'.
16090   match(If cmp cr);
16091 
16092   effect(USE lbl);
16093 
16094   ins_cost(BRANCH_COST);
16095   // If set to 1 this indicates that the current instruction is a
16096   // short variant of a long branch. This avoids using this
16097   // instruction in first-pass matching. It will then only be used in
16098   // the `Shorten_branches' pass.
16099   // ins_short_branch(1);
16100   format %{ "b$cmp  $lbl" %}
16101 
16102   ins_encode(aarch64_enc_br_con(cmp, lbl));
16103 
16104   ins_pipe(pipe_branch_cond);
16105 %}
16106 
16107 // Conditional Near Branch Unsigned
16108 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16109 %{
16110   // Same match rule as `branchConFar'.
16111   match(If cmp cr);
16112 
16113   effect(USE lbl);
16114 
16115   ins_cost(BRANCH_COST);
16116   // If set to 1 this indicates that the current instruction is a
16117   // short variant of a long branch. This avoids using this
16118   // instruction in first-pass matching. It will then only be used in
16119   // the `Shorten_branches' pass.
16120   // ins_short_branch(1);
16121   format %{ "b$cmp  $lbl\t# unsigned" %}
16122 
16123   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16124 
16125   ins_pipe(pipe_branch_cond);
16126 %}
16127 
16128 // Make use of CBZ and CBNZ.  These instructions, as well as being
16129 // shorter than (cmp; branch), have the additional benefit of not
16130 // killing the flags.
16131 
16132 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16133   match(If cmp (CmpI op1 op2));
16134   effect(USE labl);
16135 
16136   ins_cost(BRANCH_COST);
16137   format %{ "cbw$cmp   $op1, $labl" %}
16138   ins_encode %{
16139     Label* L = $labl$$label;
16140     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16141     if (cond == Assembler::EQ)
16142       __ cbzw($op1$$Register, *L);
16143     else
16144       __ cbnzw($op1$$Register, *L);
16145   %}
16146   ins_pipe(pipe_cmp_branch);
16147 %}
16148 
16149 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16150   match(If cmp (CmpL op1 op2));
16151   effect(USE labl);
16152 
16153   ins_cost(BRANCH_COST);
16154   format %{ "cb$cmp   $op1, $labl" %}
16155   ins_encode %{
16156     Label* L = $labl$$label;
16157     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16158     if (cond == Assembler::EQ)
16159       __ cbz($op1$$Register, *L);
16160     else
16161       __ cbnz($op1$$Register, *L);
16162   %}
16163   ins_pipe(pipe_cmp_branch);
16164 %}
16165 
16166 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16167   match(If cmp (CmpP op1 op2));
16168   effect(USE labl);
16169 
16170   ins_cost(BRANCH_COST);
16171   format %{ "cb$cmp   $op1, $labl" %}
16172   ins_encode %{
16173     Label* L = $labl$$label;
16174     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16175     if (cond == Assembler::EQ)
16176       __ cbz($op1$$Register, *L);
16177     else
16178       __ cbnz($op1$$Register, *L);
16179   %}
16180   ins_pipe(pipe_cmp_branch);
16181 %}
16182 
16183 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16184   match(If cmp (CmpN op1 op2));
16185   effect(USE labl);
16186 
16187   ins_cost(BRANCH_COST);
16188   format %{ "cbw$cmp   $op1, $labl" %}
16189   ins_encode %{
16190     Label* L = $labl$$label;
16191     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16192     if (cond == Assembler::EQ)
16193       __ cbzw($op1$$Register, *L);
16194     else
16195       __ cbnzw($op1$$Register, *L);
16196   %}
16197   ins_pipe(pipe_cmp_branch);
16198 %}
16199 
16200 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16201   match(If cmp (CmpP (DecodeN oop) zero));
16202   effect(USE labl);
16203 
16204   ins_cost(BRANCH_COST);
16205   format %{ "cb$cmp   $oop, $labl" %}
16206   ins_encode %{
16207     Label* L = $labl$$label;
16208     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16209     if (cond == Assembler::EQ)
16210       __ cbzw($oop$$Register, *L);
16211     else
16212       __ cbnzw($oop$$Register, *L);
16213   %}
16214   ins_pipe(pipe_cmp_branch);
16215 %}
16216 
16217 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16218   match(If cmp (CmpU op1 op2));
16219   effect(USE labl);
16220 
16221   ins_cost(BRANCH_COST);
16222   format %{ "cbw$cmp   $op1, $labl" %}
16223   ins_encode %{
16224     Label* L = $labl$$label;
16225     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16226     if (cond == Assembler::EQ || cond == Assembler::LS)
16227       __ cbzw($op1$$Register, *L);
16228     else
16229       __ cbnzw($op1$$Register, *L);
16230   %}
16231   ins_pipe(pipe_cmp_branch);
16232 %}
16233 
16234 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16235   match(If cmp (CmpUL op1 op2));
16236   effect(USE labl);
16237 
16238   ins_cost(BRANCH_COST);
16239   format %{ "cb$cmp   $op1, $labl" %}
16240   ins_encode %{
16241     Label* L = $labl$$label;
16242     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16243     if (cond == Assembler::EQ || cond == Assembler::LS)
16244       __ cbz($op1$$Register, *L);
16245     else
16246       __ cbnz($op1$$Register, *L);
16247   %}
16248   ins_pipe(pipe_cmp_branch);
16249 %}
16250 
16251 // Test bit and Branch
16252 
16253 // Patterns for short (< 32KiB) variants
16254 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16255   match(If cmp (CmpL op1 op2));
16256   effect(USE labl);
16257 
16258   ins_cost(BRANCH_COST);
16259   format %{ "cb$cmp   $op1, $labl # long" %}
16260   ins_encode %{
16261     Label* L = $labl$$label;
16262     Assembler::Condition cond =
16263       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16264     __ tbr(cond, $op1$$Register, 63, *L);
16265   %}
16266   ins_pipe(pipe_cmp_branch);
16267   ins_short_branch(1);
16268 %}
16269 
16270 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16271   match(If cmp (CmpI op1 op2));
16272   effect(USE labl);
16273 
16274   ins_cost(BRANCH_COST);
16275   format %{ "cb$cmp   $op1, $labl # int" %}
16276   ins_encode %{
16277     Label* L = $labl$$label;
16278     Assembler::Condition cond =
16279       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16280     __ tbr(cond, $op1$$Register, 31, *L);
16281   %}
16282   ins_pipe(pipe_cmp_branch);
16283   ins_short_branch(1);
16284 %}
16285 
16286 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16287   match(If cmp (CmpL (AndL op1 op2) op3));
16288   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16289   effect(USE labl);
16290 
16291   ins_cost(BRANCH_COST);
16292   format %{ "tb$cmp   $op1, $op2, $labl" %}
16293   ins_encode %{
16294     Label* L = $labl$$label;
16295     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16296     int bit = exact_log2_long($op2$$constant);
16297     __ tbr(cond, $op1$$Register, bit, *L);
16298   %}
16299   ins_pipe(pipe_cmp_branch);
16300   ins_short_branch(1);
16301 %}
16302 
16303 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16304   match(If cmp (CmpI (AndI op1 op2) op3));
16305   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16306   effect(USE labl);
16307 
16308   ins_cost(BRANCH_COST);
16309   format %{ "tb$cmp   $op1, $op2, $labl" %}
16310   ins_encode %{
16311     Label* L = $labl$$label;
16312     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16313     int bit = exact_log2((juint)$op2$$constant);
16314     __ tbr(cond, $op1$$Register, bit, *L);
16315   %}
16316   ins_pipe(pipe_cmp_branch);
16317   ins_short_branch(1);
16318 %}
16319 
16320 // And far variants
16321 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16322   match(If cmp (CmpL op1 op2));
16323   effect(USE labl);
16324 
16325   ins_cost(BRANCH_COST);
16326   format %{ "cb$cmp   $op1, $labl # long" %}
16327   ins_encode %{
16328     Label* L = $labl$$label;
16329     Assembler::Condition cond =
16330       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16331     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16332   %}
16333   ins_pipe(pipe_cmp_branch);
16334 %}
16335 
16336 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16337   match(If cmp (CmpI op1 op2));
16338   effect(USE labl);
16339 
16340   ins_cost(BRANCH_COST);
16341   format %{ "cb$cmp   $op1, $labl # int" %}
16342   ins_encode %{
16343     Label* L = $labl$$label;
16344     Assembler::Condition cond =
16345       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16346     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16347   %}
16348   ins_pipe(pipe_cmp_branch);
16349 %}
16350 
16351 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16352   match(If cmp (CmpL (AndL op1 op2) op3));
16353   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16354   effect(USE labl);
16355 
16356   ins_cost(BRANCH_COST);
16357   format %{ "tb$cmp   $op1, $op2, $labl" %}
16358   ins_encode %{
16359     Label* L = $labl$$label;
16360     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16361     int bit = exact_log2_long($op2$$constant);
16362     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16363   %}
16364   ins_pipe(pipe_cmp_branch);
16365 %}
16366 
16367 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16368   match(If cmp (CmpI (AndI op1 op2) op3));
16369   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16370   effect(USE labl);
16371 
16372   ins_cost(BRANCH_COST);
16373   format %{ "tb$cmp   $op1, $op2, $labl" %}
16374   ins_encode %{
16375     Label* L = $labl$$label;
16376     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16377     int bit = exact_log2((juint)$op2$$constant);
16378     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16379   %}
16380   ins_pipe(pipe_cmp_branch);
16381 %}
16382 
16383 // Test bits
16384 
16385 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16386   match(Set cr (CmpL (AndL op1 op2) op3));
16387   predicate(Assembler::operand_valid_for_logical_immediate
16388             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16389 
16390   ins_cost(INSN_COST);
16391   format %{ "tst $op1, $op2 # long" %}
16392   ins_encode %{
16393     __ tst($op1$$Register, $op2$$constant);
16394   %}
16395   ins_pipe(ialu_reg_reg);
16396 %}
16397 
16398 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16399   match(Set cr (CmpI (AndI op1 op2) op3));
16400   predicate(Assembler::operand_valid_for_logical_immediate
16401             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16402 
16403   ins_cost(INSN_COST);
16404   format %{ "tst $op1, $op2 # int" %}
16405   ins_encode %{
16406     __ tstw($op1$$Register, $op2$$constant);
16407   %}
16408   ins_pipe(ialu_reg_reg);
16409 %}
16410 
16411 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16412   match(Set cr (CmpL (AndL op1 op2) op3));
16413 
16414   ins_cost(INSN_COST);
16415   format %{ "tst $op1, $op2 # long" %}
16416   ins_encode %{
16417     __ tst($op1$$Register, $op2$$Register);
16418   %}
16419   ins_pipe(ialu_reg_reg);
16420 %}
16421 
16422 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16423   match(Set cr (CmpI (AndI op1 op2) op3));
16424 
16425   ins_cost(INSN_COST);
16426   format %{ "tstw $op1, $op2 # int" %}
16427   ins_encode %{
16428     __ tstw($op1$$Register, $op2$$Register);
16429   %}
16430   ins_pipe(ialu_reg_reg);
16431 %}
16432 
16433 
16434 // Conditional Far Branch
16435 // Conditional Far Branch Unsigned
16436 // TODO: fixme
16437 
16438 // counted loop end branch near
16439 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16440 %{
16441   match(CountedLoopEnd cmp cr);
16442 
16443   effect(USE lbl);
16444 
16445   ins_cost(BRANCH_COST);
16446   // short variant.
16447   // ins_short_branch(1);
16448   format %{ "b$cmp $lbl \t// counted loop end" %}
16449 
16450   ins_encode(aarch64_enc_br_con(cmp, lbl));
16451 
16452   ins_pipe(pipe_branch);
16453 %}
16454 
16455 // counted loop end branch near Unsigned
16456 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16457 %{
16458   match(CountedLoopEnd cmp cr);
16459 
16460   effect(USE lbl);
16461 
16462   ins_cost(BRANCH_COST);
16463   // short variant.
16464   // ins_short_branch(1);
16465   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16466 
16467   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16468 
16469   ins_pipe(pipe_branch);
16470 %}
16471 
16472 // counted loop end branch far
16473 // counted loop end branch far unsigned
16474 // TODO: fixme
16475 
16476 // ============================================================================
16477 // inlined locking and unlocking
16478 
16479 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16480 %{
16481   match(Set cr (FastLock object box));
16482   effect(TEMP tmp, TEMP tmp2);
16483 
16484   // TODO
16485   // identify correct cost
16486   ins_cost(5 * INSN_COST);
16487   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16488 
16489   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16490 
16491   ins_pipe(pipe_serial);
16492 %}
16493 
16494 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16495 %{
16496   match(Set cr (FastUnlock object box));
16497   effect(TEMP tmp, TEMP tmp2);
16498 
16499   ins_cost(5 * INSN_COST);
16500   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16501 
16502   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16503 
16504   ins_pipe(pipe_serial);
16505 %}
16506 
16507 
16508 // ============================================================================
16509 // Safepoint Instructions
16510 
16511 // TODO
16512 // provide a near and far version of this code
16513 
16514 instruct safePoint(rFlagsReg cr, iRegP poll)
16515 %{
16516   match(SafePoint poll);
16517   effect(KILL cr);
16518 
16519   format %{
16520     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16521   %}
16522   ins_encode %{
16523     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16524   %}
16525   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16526 %}
16527 
16528 
16529 // ============================================================================
16530 // Procedure Call/Return Instructions
16531 
16532 // Call Java Static Instruction
16533 
16534 instruct CallStaticJavaDirect(method meth)
16535 %{
16536   match(CallStaticJava);
16537 
16538   effect(USE meth);
16539 
16540   ins_cost(CALL_COST);
16541 
16542   format %{ "call,static $meth \t// ==> " %}
16543 
16544   ins_encode(aarch64_enc_java_static_call(meth),
16545              aarch64_enc_call_epilog);
16546 
16547   ins_pipe(pipe_class_call);
16548 %}
16549 
16550 // TO HERE
16551 
16552 // Call Java Dynamic Instruction
16553 instruct CallDynamicJavaDirect(method meth)
16554 %{
16555   match(CallDynamicJava);
16556 
16557   effect(USE meth);
16558 
16559   ins_cost(CALL_COST);
16560 
16561   format %{ "CALL,dynamic $meth \t// ==> " %}
16562 
16563   ins_encode(aarch64_enc_java_dynamic_call(meth),
16564              aarch64_enc_call_epilog);
16565 
16566   ins_pipe(pipe_class_call);
16567 %}
16568 
16569 // Call Runtime Instruction
16570 
16571 instruct CallRuntimeDirect(method meth)
16572 %{
16573   match(CallRuntime);
16574 
16575   effect(USE meth);
16576 
16577   ins_cost(CALL_COST);
16578 
16579   format %{ "CALL, runtime $meth" %}
16580 
16581   ins_encode( aarch64_enc_java_to_runtime(meth) );
16582 
16583   ins_pipe(pipe_class_call);
16584 %}
16585 
16586 // Call Runtime Instruction
16587 
16588 instruct CallLeafDirect(method meth)
16589 %{
16590   match(CallLeaf);
16591 
16592   effect(USE meth);
16593 
16594   ins_cost(CALL_COST);
16595 
16596   format %{ "CALL, runtime leaf $meth" %}
16597 
16598   ins_encode( aarch64_enc_java_to_runtime(meth) );
16599 
16600   ins_pipe(pipe_class_call);
16601 %}
16602 
16603 // Call Runtime Instruction
16604 
16605 instruct CallLeafNoFPDirect(method meth)
16606 %{
16607   match(CallLeafNoFP);
16608 
16609   effect(USE meth);
16610 
16611   ins_cost(CALL_COST);
16612 
16613   format %{ "CALL, runtime leaf nofp $meth" %}
16614 
16615   ins_encode( aarch64_enc_java_to_runtime(meth) );
16616 
16617   ins_pipe(pipe_class_call);
16618 %}
16619 
16620 instruct CallNativeDirect(method meth)
16621 %{
16622   match(CallNative);
16623 
16624   effect(USE meth);
16625 
16626   ins_cost(CALL_COST);
16627 
16628   format %{ "CALL, native $meth" %}
16629 
16630   ins_encode( aarch64_enc_java_to_runtime(meth) );
16631 
16632   ins_pipe(pipe_class_call);
16633 %}
16634 
16635 // Tail Call; Jump from runtime stub to Java code.
16636 // Also known as an 'interprocedural jump'.
16637 // Target of jump will eventually return to caller.
16638 // TailJump below removes the return address.
16639 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16640 %{
16641   match(TailCall jump_target method_ptr);
16642 
16643   ins_cost(CALL_COST);
16644 
16645   format %{ "br $jump_target\t# $method_ptr holds method" %}
16646 
16647   ins_encode(aarch64_enc_tail_call(jump_target));
16648 
16649   ins_pipe(pipe_class_call);
16650 %}
16651 
16652 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16653 %{
16654   match(TailJump jump_target ex_oop);
16655 
16656   ins_cost(CALL_COST);
16657 
16658   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16659 
16660   ins_encode(aarch64_enc_tail_jmp(jump_target));
16661 
16662   ins_pipe(pipe_class_call);
16663 %}
16664 
16665 // Create exception oop: created by stack-crawling runtime code.
16666 // Created exception is now available to this handler, and is setup
16667 // just prior to jumping to this handler. No code emitted.
16668 // TODO check
16669 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16670 instruct CreateException(iRegP_R0 ex_oop)
16671 %{
16672   match(Set ex_oop (CreateEx));
16673 
16674   format %{ " -- \t// exception oop; no code emitted" %}
16675 
16676   size(0);
16677 
16678   ins_encode( /*empty*/ );
16679 
16680   ins_pipe(pipe_class_empty);
16681 %}
16682 
16683 // Rethrow exception: The exception oop will come in the first
16684 // argument position. Then JUMP (not call) to the rethrow stub code.
16685 instruct RethrowException() %{
16686   match(Rethrow);
16687   ins_cost(CALL_COST);
16688 
16689   format %{ "b rethrow_stub" %}
16690 
16691   ins_encode( aarch64_enc_rethrow() );
16692 
16693   ins_pipe(pipe_class_call);
16694 %}
16695 
16696 
16697 // Return Instruction
16698 // epilog node loads ret address into lr as part of frame pop
16699 instruct Ret()
16700 %{
16701   match(Return);
16702 
16703   format %{ "ret\t// return register" %}
16704 
16705   ins_encode( aarch64_enc_ret() );
16706 
16707   ins_pipe(pipe_branch);
16708 %}
16709 
16710 // Die now.
16711 instruct ShouldNotReachHere() %{
16712   match(Halt);
16713 
16714   ins_cost(CALL_COST);
16715   format %{ "ShouldNotReachHere" %}
16716 
16717   ins_encode %{
16718     if (is_reachable()) {
16719       __ stop(_halt_reason);
16720     }
16721   %}
16722 
16723   ins_pipe(pipe_class_default);
16724 %}
16725 
16726 // ============================================================================
16727 // Partial Subtype Check
16728 //
16729 // superklass array for an instance of the superklass.  Set a hidden
16730 // internal cache on a hit (cache is checked with exposed code in
16731 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16732 // encoding ALSO sets flags.
16733 
16734 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16735 %{
16736   match(Set result (PartialSubtypeCheck sub super));
16737   effect(KILL cr, KILL temp);
16738 
16739   ins_cost(1100);  // slightly larger than the next version
16740   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16741 
16742   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16743 
16744   opcode(0x1); // Force zero of result reg on hit
16745 
16746   ins_pipe(pipe_class_memory);
16747 %}
16748 
16749 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16750 %{
16751   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16752   effect(KILL temp, KILL result);
16753 
16754   ins_cost(1100);  // slightly larger than the next version
16755   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16756 
16757   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16758 
16759   opcode(0x0); // Don't zero result reg on hit
16760 
16761   ins_pipe(pipe_class_memory);
16762 %}
16763 
16764 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16765                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16766 %{
16767   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16768   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16769   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16770 
16771   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16772   ins_encode %{
16773     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16774     __ string_compare($str1$$Register, $str2$$Register,
16775                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16776                       $tmp1$$Register, $tmp2$$Register,
16777                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16778   %}
16779   ins_pipe(pipe_class_memory);
16780 %}
16781 
16782 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16783                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16784 %{
16785   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16786   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16787   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16788 
16789   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16790   ins_encode %{
16791     __ string_compare($str1$$Register, $str2$$Register,
16792                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16793                       $tmp1$$Register, $tmp2$$Register,
16794                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16795   %}
16796   ins_pipe(pipe_class_memory);
16797 %}
16798 
16799 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16800                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16801                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16802 %{
16803   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16804   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16805   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16806          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16807 
16808   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16809   ins_encode %{
16810     __ string_compare($str1$$Register, $str2$$Register,
16811                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16812                       $tmp1$$Register, $tmp2$$Register,
16813                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16814                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16815   %}
16816   ins_pipe(pipe_class_memory);
16817 %}
16818 
16819 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16820                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16821                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16822 %{
16823   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16824   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16825   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16826          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16827 
16828   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16829   ins_encode %{
16830     __ string_compare($str1$$Register, $str2$$Register,
16831                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16832                       $tmp1$$Register, $tmp2$$Register,
16833                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16834                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16835   %}
16836   ins_pipe(pipe_class_memory);
16837 %}
16838 
16839 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16840        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16841        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16842 %{
16843   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16844   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16845   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16846          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16847   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16848 
16849   ins_encode %{
16850     __ string_indexof($str1$$Register, $str2$$Register,
16851                       $cnt1$$Register, $cnt2$$Register,
16852                       $tmp1$$Register, $tmp2$$Register,
16853                       $tmp3$$Register, $tmp4$$Register,
16854                       $tmp5$$Register, $tmp6$$Register,
16855                       -1, $result$$Register, StrIntrinsicNode::UU);
16856   %}
16857   ins_pipe(pipe_class_memory);
16858 %}
16859 
16860 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16861        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16862        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16863 %{
16864   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16865   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16866   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16867          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16868   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16869 
16870   ins_encode %{
16871     __ string_indexof($str1$$Register, $str2$$Register,
16872                       $cnt1$$Register, $cnt2$$Register,
16873                       $tmp1$$Register, $tmp2$$Register,
16874                       $tmp3$$Register, $tmp4$$Register,
16875                       $tmp5$$Register, $tmp6$$Register,
16876                       -1, $result$$Register, StrIntrinsicNode::LL);
16877   %}
16878   ins_pipe(pipe_class_memory);
16879 %}
16880 
16881 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16882        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16883        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16884 %{
16885   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16886   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16887   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16888          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16889   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16890 
16891   ins_encode %{
16892     __ string_indexof($str1$$Register, $str2$$Register,
16893                       $cnt1$$Register, $cnt2$$Register,
16894                       $tmp1$$Register, $tmp2$$Register,
16895                       $tmp3$$Register, $tmp4$$Register,
16896                       $tmp5$$Register, $tmp6$$Register,
16897                       -1, $result$$Register, StrIntrinsicNode::UL);
16898   %}
16899   ins_pipe(pipe_class_memory);
16900 %}
16901 
16902 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16903                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16904                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16905 %{
16906   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16907   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16908   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16909          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16910   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16911 
16912   ins_encode %{
16913     int icnt2 = (int)$int_cnt2$$constant;
16914     __ string_indexof($str1$$Register, $str2$$Register,
16915                       $cnt1$$Register, zr,
16916                       $tmp1$$Register, $tmp2$$Register,
16917                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16918                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16919   %}
16920   ins_pipe(pipe_class_memory);
16921 %}
16922 
16923 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16924                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16925                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16926 %{
16927   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16928   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16929   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16930          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16931   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16932 
16933   ins_encode %{
16934     int icnt2 = (int)$int_cnt2$$constant;
16935     __ string_indexof($str1$$Register, $str2$$Register,
16936                       $cnt1$$Register, zr,
16937                       $tmp1$$Register, $tmp2$$Register,
16938                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16939                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16940   %}
16941   ins_pipe(pipe_class_memory);
16942 %}
16943 
16944 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16945                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16946                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16947 %{
16948   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16949   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16950   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16951          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16952   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16953 
16954   ins_encode %{
16955     int icnt2 = (int)$int_cnt2$$constant;
16956     __ string_indexof($str1$$Register, $str2$$Register,
16957                       $cnt1$$Register, zr,
16958                       $tmp1$$Register, $tmp2$$Register,
16959                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16960                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16961   %}
16962   ins_pipe(pipe_class_memory);
16963 %}
16964 
16965 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16966                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16967                               iRegINoSp tmp3, rFlagsReg cr)
16968 %{
16969   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16970   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16971   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16972          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16973 
16974   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16975 
16976   ins_encode %{
16977     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16978                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16979                            $tmp3$$Register);
16980   %}
16981   ins_pipe(pipe_class_memory);
16982 %}
16983 
16984 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16985                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16986                               iRegINoSp tmp3, rFlagsReg cr)
16987 %{
16988   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16989   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16990   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16991          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16992 
16993   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16994 
16995   ins_encode %{
16996     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16997                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16998                            $tmp3$$Register);
16999   %}
17000   ins_pipe(pipe_class_memory);
17001 %}
17002 
17003 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17004                         iRegI_R0 result, rFlagsReg cr)
17005 %{
17006   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17007   match(Set result (StrEquals (Binary str1 str2) cnt));
17008   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17009 
17010   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17011   ins_encode %{
17012     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17013     __ string_equals($str1$$Register, $str2$$Register,
17014                      $result$$Register, $cnt$$Register, 1);
17015   %}
17016   ins_pipe(pipe_class_memory);
17017 %}
17018 
17019 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17020                         iRegI_R0 result, rFlagsReg cr)
17021 %{
17022   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17023   match(Set result (StrEquals (Binary str1 str2) cnt));
17024   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17025 
17026   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17027   ins_encode %{
17028     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17029     __ string_equals($str1$$Register, $str2$$Register,
17030                      $result$$Register, $cnt$$Register, 2);
17031   %}
17032   ins_pipe(pipe_class_memory);
17033 %}
17034 
17035 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17036                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17037                        iRegP_R10 tmp, rFlagsReg cr)
17038 %{
17039   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17040   match(Set result (AryEq ary1 ary2));
17041   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17042 
17043   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17044   ins_encode %{
17045     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17046                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17047                                    $result$$Register, $tmp$$Register, 1);
17048     if (tpc == NULL) {
17049       ciEnv::current()->record_failure("CodeCache is full");
17050       return;
17051     }
17052   %}
17053   ins_pipe(pipe_class_memory);
17054 %}
17055 
17056 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17057                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17058                        iRegP_R10 tmp, rFlagsReg cr)
17059 %{
17060   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17061   match(Set result (AryEq ary1 ary2));
17062   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17063 
17064   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17065   ins_encode %{
17066     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17067                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17068                                    $result$$Register, $tmp$$Register, 2);
17069     if (tpc == NULL) {
17070       ciEnv::current()->record_failure("CodeCache is full");
17071       return;
17072     }
17073   %}
17074   ins_pipe(pipe_class_memory);
17075 %}
17076 
17077 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17078 %{
17079   match(Set result (HasNegatives ary1 len));
17080   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17081   format %{ "has negatives byte[] $ary1,$len -> $result" %}
17082   ins_encode %{
17083     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17084     if (tpc == NULL) {
17085       ciEnv::current()->record_failure("CodeCache is full");
17086       return;
17087     }
17088   %}
17089   ins_pipe( pipe_slow );
17090 %}
17091 
17092 // fast char[] to byte[] compression
17093 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17094                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17095                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17096                          iRegI_R0 result, rFlagsReg cr)
17097 %{
17098   match(Set result (StrCompressedCopy src (Binary dst len)));
17099   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17100 
17101   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
17102   ins_encode %{
17103     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17104                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17105                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17106                            $result$$Register);
17107   %}
17108   ins_pipe( pipe_slow );
17109 %}
17110 
17111 // fast byte[] to char[] inflation
17112 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17113                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17114 %{
17115   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17116   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17117 
17118   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17119   ins_encode %{
17120     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17121                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17122                                         $tmp3$$FloatRegister, $tmp4$$Register);
17123     if (tpc == NULL) {
17124       ciEnv::current()->record_failure("CodeCache is full");
17125       return;
17126     }
17127   %}
17128   ins_pipe(pipe_class_memory);
17129 %}
17130 
17131 // encode char[] to byte[] in ISO_8859_1
17132 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17133                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17134                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17135                           iRegI_R0 result, rFlagsReg cr)
17136 %{
17137   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17138   match(Set result (EncodeISOArray src (Binary dst len)));
17139   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17140          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17141 
17142   format %{ "Encode array $src,$dst,$len -> $result" %}
17143   ins_encode %{
17144     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17145          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
17146          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
17147   %}
17148   ins_pipe( pipe_class_memory );
17149 %}
17150 
17151 // ============================================================================
17152 // This name is KNOWN by the ADLC and cannot be changed.
17153 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17154 // for this guy.
17155 instruct tlsLoadP(thread_RegP dst)
17156 %{
17157   match(Set dst (ThreadLocal));
17158 
17159   ins_cost(0);
17160 
17161   format %{ " -- \t// $dst=Thread::current(), empty" %}
17162 
17163   size(0);
17164 
17165   ins_encode( /*empty*/ );
17166 
17167   ins_pipe(pipe_class_empty);
17168 %}
17169 
17170 //----------PEEPHOLE RULES-----------------------------------------------------
17171 // These must follow all instruction definitions as they use the names
17172 // defined in the instructions definitions.
17173 //
17174 // peepmatch ( root_instr_name [preceding_instruction]* );
17175 //
17176 // peepconstraint %{
17177 // (instruction_number.operand_name relational_op instruction_number.operand_name
17178 //  [, ...] );
17179 // // instruction numbers are zero-based using left to right order in peepmatch
17180 //
17181 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17182 // // provide an instruction_number.operand_name for each operand that appears
17183 // // in the replacement instruction's match rule
17184 //
17185 // ---------VM FLAGS---------------------------------------------------------
17186 //
17187 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17188 //
17189 // Each peephole rule is given an identifying number starting with zero and
17190 // increasing by one in the order seen by the parser.  An individual peephole
17191 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17192 // on the command-line.
17193 //
17194 // ---------CURRENT LIMITATIONS----------------------------------------------
17195 //
17196 // Only match adjacent instructions in same basic block
17197 // Only equality constraints
17198 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17199 // Only one replacement instruction
17200 //
17201 // ---------EXAMPLE----------------------------------------------------------
17202 //
17203 // // pertinent parts of existing instructions in architecture description
17204 // instruct movI(iRegINoSp dst, iRegI src)
17205 // %{
17206 //   match(Set dst (CopyI src));
17207 // %}
17208 //
17209 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17210 // %{
17211 //   match(Set dst (AddI dst src));
17212 //   effect(KILL cr);
17213 // %}
17214 //
17215 // // Change (inc mov) to lea
17216 // peephole %{
17217 //   // increment preceeded by register-register move
17218 //   peepmatch ( incI_iReg movI );
17219 //   // require that the destination register of the increment
17220 //   // match the destination register of the move
17221 //   peepconstraint ( 0.dst == 1.dst );
17222 //   // construct a replacement instruction that sets
17223 //   // the destination to ( move's source register + one )
17224 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17225 // %}
17226 //
17227 
17228 // Implementation no longer uses movX instructions since
17229 // machine-independent system no longer uses CopyX nodes.
17230 //
17231 // peephole
17232 // %{
17233 //   peepmatch (incI_iReg movI);
17234 //   peepconstraint (0.dst == 1.dst);
17235 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17236 // %}
17237 
17238 // peephole
17239 // %{
17240 //   peepmatch (decI_iReg movI);
17241 //   peepconstraint (0.dst == 1.dst);
17242 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17243 // %}
17244 
17245 // peephole
17246 // %{
17247 //   peepmatch (addI_iReg_imm movI);
17248 //   peepconstraint (0.dst == 1.dst);
17249 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17250 // %}
17251 
17252 // peephole
17253 // %{
17254 //   peepmatch (incL_iReg movL);
17255 //   peepconstraint (0.dst == 1.dst);
17256 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17257 // %}
17258 
17259 // peephole
17260 // %{
17261 //   peepmatch (decL_iReg movL);
17262 //   peepconstraint (0.dst == 1.dst);
17263 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17264 // %}
17265 
17266 // peephole
17267 // %{
17268 //   peepmatch (addL_iReg_imm movL);
17269 //   peepconstraint (0.dst == 1.dst);
17270 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17271 // %}
17272 
17273 // peephole
17274 // %{
17275 //   peepmatch (addP_iReg_imm movP);
17276 //   peepconstraint (0.dst == 1.dst);
17277 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17278 // %}
17279 
17280 // // Change load of spilled value to only a spill
17281 // instruct storeI(memory mem, iRegI src)
17282 // %{
17283 //   match(Set mem (StoreI mem src));
17284 // %}
17285 //
17286 // instruct loadI(iRegINoSp dst, memory mem)
17287 // %{
17288 //   match(Set dst (LoadI mem));
17289 // %}
17290 //
17291 
17292 //----------SMARTSPILL RULES---------------------------------------------------
17293 // These must follow all instruction definitions as they use the names
17294 // defined in the instructions definitions.
17295 
17296 // Local Variables:
17297 // mode: c++
17298 // End: