1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298  bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() >= 16) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1990       C->output()->add_stub(stub);
 1991       code_stub = &stub->entry();
 1992     }
 1993     __ relocate(relocInfo::poll_return_type);
 1994     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1995   }
 1996 }
 1997 
 1998 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1999   // Variable size. Determine dynamically.
 2000   return MachNode::size(ra_);
 2001 }
 2002 
 2003 int MachEpilogNode::reloc() const {
 2004   // Return number of relocatable values contained in this instruction.
 2005   return 1; // 1 for polling page.
 2006 }
 2007 
 2008 const Pipeline * MachEpilogNode::pipeline() const {
 2009   return MachNode::pipeline_class();
 2010 }
 2011 
 2012 //=============================================================================
 2013 
 2014 // Figure out which register class each belongs in: rc_int, rc_float or
 2015 // rc_stack.
 2016 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2017 
 2018 static enum RC rc_class(OptoReg::Name reg) {
 2019 
 2020   if (reg == OptoReg::Bad) {
 2021     return rc_bad;
 2022   }
 2023 
 2024   // we have 32 int registers * 2 halves
 2025   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2026 
 2027   if (reg < slots_of_int_registers) {
 2028     return rc_int;
 2029   }
 2030 
 2031   // we have 32 float register * 8 halves
 2032   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2033   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2034     return rc_float;
 2035   }
 2036 
 2037   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2038   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2039     return rc_predicate;
 2040   }
 2041 
 2042   // Between predicate regs & stack is the flags.
 2043   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2044 
 2045   return rc_stack;
 2046 }
 2047 
 2048 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2049   Compile* C = ra_->C;
 2050 
 2051   // Get registers to move.
 2052   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2053   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2054   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2055   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2056 
 2057   enum RC src_hi_rc = rc_class(src_hi);
 2058   enum RC src_lo_rc = rc_class(src_lo);
 2059   enum RC dst_hi_rc = rc_class(dst_hi);
 2060   enum RC dst_lo_rc = rc_class(dst_lo);
 2061 
 2062   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2063 
 2064   if (src_hi != OptoReg::Bad) {
 2065     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2066            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2067            "expected aligned-adjacent pairs");
 2068   }
 2069 
 2070   if (src_lo == dst_lo && src_hi == dst_hi) {
 2071     return 0;            // Self copy, no move.
 2072   }
 2073 
 2074   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2075               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2076   int src_offset = ra_->reg2offset(src_lo);
 2077   int dst_offset = ra_->reg2offset(dst_lo);
 2078 
 2079   if (bottom_type()->isa_vect() != NULL) {
 2080     uint ireg = ideal_reg();
 2081     if (ireg == Op_VecA && cbuf) {
 2082       C2_MacroAssembler _masm(cbuf);
 2083       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2084       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2085         // stack->stack
 2086         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2087                                                 sve_vector_reg_size_in_bytes);
 2088       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2089         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2090                             sve_vector_reg_size_in_bytes);
 2091       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2092         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2093                               sve_vector_reg_size_in_bytes);
 2094       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2095         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2096                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2097                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2098       } else {
 2099         ShouldNotReachHere();
 2100       }
 2101     } else if (cbuf) {
 2102       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2103       C2_MacroAssembler _masm(cbuf);
 2104       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2105       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2106         // stack->stack
 2107         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2108         if (ireg == Op_VecD) {
 2109           __ unspill(rscratch1, true, src_offset);
 2110           __ spill(rscratch1, true, dst_offset);
 2111         } else {
 2112           __ spill_copy128(src_offset, dst_offset);
 2113         }
 2114       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2115         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2116                ireg == Op_VecD ? __ T8B : __ T16B,
 2117                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2118       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2119         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2120                  ireg == Op_VecD ? __ D : __ Q,
 2121                  ra_->reg2offset(dst_lo));
 2122       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2123         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2124                    ireg == Op_VecD ? __ D : __ Q,
 2125                    ra_->reg2offset(src_lo));
 2126       } else {
 2127         ShouldNotReachHere();
 2128       }
 2129     }
 2130   } else if (cbuf) {
 2131     C2_MacroAssembler _masm(cbuf);
 2132     switch (src_lo_rc) {
 2133     case rc_int:
 2134       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2135         if (is64) {
 2136             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2137                    as_Register(Matcher::_regEncode[src_lo]));
 2138         } else {
 2139             C2_MacroAssembler _masm(cbuf);
 2140             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2141                     as_Register(Matcher::_regEncode[src_lo]));
 2142         }
 2143       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2144         if (is64) {
 2145             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2146                      as_Register(Matcher::_regEncode[src_lo]));
 2147         } else {
 2148             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2149                      as_Register(Matcher::_regEncode[src_lo]));
 2150         }
 2151       } else {                    // gpr --> stack spill
 2152         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2153         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2154       }
 2155       break;
 2156     case rc_float:
 2157       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2158         if (is64) {
 2159             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2160                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2161         } else {
 2162             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2163                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2164         }
 2165       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2166           if (cbuf) {
 2167             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2168                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2169         } else {
 2170             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2171                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2172         }
 2173       } else {                    // fpr --> stack spill
 2174         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2175         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2176                  is64 ? __ D : __ S, dst_offset);
 2177       }
 2178       break;
 2179     case rc_stack:
 2180       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2181         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2182       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2183         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2184                    is64 ? __ D : __ S, src_offset);
 2185       } else {                    // stack --> stack copy
 2186         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2187         __ unspill(rscratch1, is64, src_offset);
 2188         __ spill(rscratch1, is64, dst_offset);
 2189       }
 2190       break;
 2191     default:
 2192       assert(false, "bad rc_class for spill");
 2193       ShouldNotReachHere();
 2194     }
 2195   }
 2196 
 2197   if (st) {
 2198     st->print("spill ");
 2199     if (src_lo_rc == rc_stack) {
 2200       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2201     } else {
 2202       st->print("%s -> ", Matcher::regName[src_lo]);
 2203     }
 2204     if (dst_lo_rc == rc_stack) {
 2205       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2206     } else {
 2207       st->print("%s", Matcher::regName[dst_lo]);
 2208     }
 2209     if (bottom_type()->isa_vect() != NULL) {
 2210       int vsize = 0;
 2211       switch (ideal_reg()) {
 2212       case Op_VecD:
 2213         vsize = 64;
 2214         break;
 2215       case Op_VecX:
 2216         vsize = 128;
 2217         break;
 2218       case Op_VecA:
 2219         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2220         break;
 2221       default:
 2222         assert(false, "bad register type for spill");
 2223         ShouldNotReachHere();
 2224       }
 2225       st->print("\t# vector spill size = %d", vsize);
 2226     } else {
 2227       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2228     }
 2229   }
 2230 
 2231   return 0;
 2232 
 2233 }
 2234 
 2235 #ifndef PRODUCT
 2236 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2237   if (!ra_)
 2238     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2239   else
 2240     implementation(NULL, ra_, false, st);
 2241 }
 2242 #endif
 2243 
 2244 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2245   implementation(&cbuf, ra_, false, NULL);
 2246 }
 2247 
 2248 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2249   return MachNode::size(ra_);
 2250 }
 2251 
 2252 //=============================================================================
 2253 
 2254 #ifndef PRODUCT
 2255 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2256   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2257   int reg = ra_->get_reg_first(this);
 2258   st->print("add %s, rsp, #%d]\t# box lock",
 2259             Matcher::regName[reg], offset);
 2260 }
 2261 #endif
 2262 
 2263 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2264   C2_MacroAssembler _masm(&cbuf);
 2265 
 2266   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2267   int reg    = ra_->get_encode(this);
 2268 
 2269   // This add will handle any 24-bit signed offset. 24 bits allows an
 2270   // 8 megabyte stack frame.
 2271   __ add(as_Register(reg), sp, offset);
 2272 }
 2273 
 2274 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2275   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2276   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2277 
 2278   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2279     return NativeInstruction::instruction_size;
 2280   } else {
 2281     return 2 * NativeInstruction::instruction_size;
 2282   }
 2283 }
 2284 
 2285 //=============================================================================
 2286 
 2287 #ifndef PRODUCT
 2288 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2289 {
 2290   st->print_cr("# MachUEPNode");
 2291   if (UseCompressedClassPointers) {
 2292     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2293     if (CompressedKlassPointers::shift() != 0) {
 2294       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2295     }
 2296   } else {
 2297    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2298   }
 2299   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2300   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2301 }
 2302 #endif
 2303 
 2304 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2305 {
 2306   // This is the unverified entry point.
 2307   C2_MacroAssembler _masm(&cbuf);
 2308 
 2309   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2310   Label skip;
 2311   // TODO
 2312   // can we avoid this skip and still use a reloc?
 2313   __ br(Assembler::EQ, skip);
 2314   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2315   __ bind(skip);
 2316 }
 2317 
 2318 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2319 {
 2320   return MachNode::size(ra_);
 2321 }
 2322 
 2323 // REQUIRED EMIT CODE
 2324 
 2325 //=============================================================================
 2326 
 2327 // Emit exception handler code.
 2328 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2329 {
 2330   // mov rscratch1 #exception_blob_entry_point
 2331   // br rscratch1
 2332   // Note that the code buffer's insts_mark is always relative to insts.
 2333   // That's why we must use the macroassembler to generate a handler.
 2334   C2_MacroAssembler _masm(&cbuf);
 2335   address base = __ start_a_stub(size_exception_handler());
 2336   if (base == NULL) {
 2337     ciEnv::current()->record_failure("CodeCache is full");
 2338     return 0;  // CodeBuffer::expand failed
 2339   }
 2340   int offset = __ offset();
 2341   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2342   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2343   __ end_a_stub();
 2344   return offset;
 2345 }
 2346 
 2347 // Emit deopt handler code.
 2348 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2349 {
 2350   // Note that the code buffer's insts_mark is always relative to insts.
 2351   // That's why we must use the macroassembler to generate a handler.
 2352   C2_MacroAssembler _masm(&cbuf);
 2353   address base = __ start_a_stub(size_deopt_handler());
 2354   if (base == NULL) {
 2355     ciEnv::current()->record_failure("CodeCache is full");
 2356     return 0;  // CodeBuffer::expand failed
 2357   }
 2358   int offset = __ offset();
 2359 
 2360   __ adr(lr, __ pc());
 2361   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2362 
 2363   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2364   __ end_a_stub();
 2365   return offset;
 2366 }
 2367 
 2368 // REQUIRED MATCHER CODE
 2369 
 2370 //=============================================================================
 2371 
 2372 const bool Matcher::match_rule_supported(int opcode) {
 2373   if (!has_match_rule(opcode))
 2374     return false;
 2375 
 2376   bool ret_value = true;
 2377   switch (opcode) {
 2378     case Op_OnSpinWait:
 2379       return VM_Version::supports_on_spin_wait();
 2380     case Op_CacheWB:
 2381     case Op_CacheWBPreSync:
 2382     case Op_CacheWBPostSync:
 2383       if (!VM_Version::supports_data_cache_line_flush()) {
 2384         ret_value = false;
 2385       }
 2386       break;
 2387   }
 2388 
 2389   return ret_value; // Per default match rules are supported.
 2390 }
 2391 
 2392 // Identify extra cases that we might want to provide match rules for vector nodes and
 2393 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2394 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2395   if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
 2396     return false;
 2397   }
 2398   int bit_size = vlen * type2aelembytes(bt) * 8;
 2399   if (UseSVE == 0 && bit_size > 128) {
 2400     return false;
 2401   }
 2402   if (UseSVE > 0) {
 2403     return op_sve_supported(opcode);
 2404   } else { // NEON
 2405     // Special cases
 2406     switch (opcode) {
 2407     case Op_VectorMaskCmp:
 2408     // We don't have VectorReinterpret with bit_size less than 64 support for
 2409     // now, even for byte type. To be refined with fully VectorCast support.
 2410     case Op_VectorReinterpret:
 2411       if (vlen < 2 || bit_size < 64) {
 2412         return false;
 2413       }
 2414       break;
 2415     case Op_MulAddVS2VI:
 2416       if (bit_size < 128) {
 2417         return false;
 2418       }
 2419       break;
 2420     case Op_MulVL:
 2421       return false;
 2422     case Op_VectorLoadShuffle:
 2423     case Op_VectorRearrange:
 2424       if (vlen < 4) {
 2425         return false;
 2426       }
 2427       break;
 2428     // Some types of VectorCast are not implemented for now.
 2429     case Op_VectorCastI2X:
 2430       if (bt == T_BYTE) {
 2431         return false;
 2432       }
 2433       break;
 2434     case Op_VectorCastS2X:
 2435       if (vlen < 4 || bit_size < 64) {
 2436         return false;
 2437       }
 2438       break;
 2439     case Op_VectorCastF2X:
 2440     case Op_VectorCastD2X:
 2441       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2442         return false;
 2443       }
 2444       break;
 2445     default:
 2446       break;
 2447     }
 2448   }
 2449   return true; // Per default match rules are supported.
 2450 }
 2451 
 2452 const RegMask* Matcher::predicate_reg_mask(void) {
 2453   return &_PR_REG_mask;
 2454 }
 2455 
 2456 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2457   return new TypeVectMask(elemTy, length);
 2458 }
 2459 
 2460 // Vector calling convention not yet implemented.
 2461 const bool Matcher::supports_vector_calling_convention(void) {
 2462   return false;
 2463 }
 2464 
 2465 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2466   Unimplemented();
 2467   return OptoRegPair(0, 0);
 2468 }
 2469 
 2470 const int Matcher::float_pressure(int default_pressure_threshold) {
 2471   return default_pressure_threshold;
 2472 }
 2473 
 2474 // Is this branch offset short enough that a short branch can be used?
 2475 //
 2476 // NOTE: If the platform does not provide any short branch variants, then
 2477 //       this method should return false for offset 0.
 2478 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2479   // The passed offset is relative to address of the branch.
 2480 
 2481   return (-32768 <= offset && offset < 32768);
 2482 }
 2483 
 2484 // Vector width in bytes.
 2485 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2486   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2487   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2488   // Minimum 2 values in vector
 2489   if (size < 2*type2aelembytes(bt)) size = 0;
 2490   // But never < 4
 2491   if (size < 4) size = 0;
 2492   return size;
 2493 }
 2494 
 2495 // Limits on vector size (number of elements) loaded into vector.
 2496 const int Matcher::max_vector_size(const BasicType bt) {
 2497   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2498 }
 2499 const int Matcher::min_vector_size(const BasicType bt) {
 2500   int max_size = max_vector_size(bt);
 2501   if ((UseSVE > 0) && (MaxVectorSize >= 16)) {
 2502     // Currently vector length less than SVE vector register size is not supported.
 2503     return max_size;
 2504   } else { // NEON
 2505     // Limit the vector size to 8 bytes
 2506     int size = 8 / type2aelembytes(bt);
 2507     if (bt == T_BYTE) {
 2508       // To support vector api shuffle/rearrange.
 2509       size = 4;
 2510     } else if (bt == T_BOOLEAN) {
 2511       // To support vector api load/store mask.
 2512       size = 2;
 2513     }
 2514     if (size < 2) size = 2;
 2515     return MIN2(size,max_size);
 2516   }
 2517 }
 2518 
 2519 // Actual max scalable vector register length.
 2520 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2521   return Matcher::max_vector_size(bt);
 2522 }
 2523 
 2524 // Vector ideal reg.
 2525 const uint Matcher::vector_ideal_reg(int len) {
 2526   if (UseSVE > 0 && 16 <= len && len <= 256) {
 2527     return Op_VecA;
 2528   }
 2529   switch(len) {
 2530     // For 16-bit/32-bit mask vector, reuse VecD.
 2531     case  2:
 2532     case  4:
 2533     case  8: return Op_VecD;
 2534     case 16: return Op_VecX;
 2535   }
 2536   ShouldNotReachHere();
 2537   return 0;
 2538 }
 2539 
 2540 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2541   ShouldNotReachHere(); // generic vector operands not supported
 2542   return NULL;
 2543 }
 2544 
 2545 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
 2546   ShouldNotReachHere();  // generic vector operands not supported
 2547   return false;
 2548 }
 2549 
 2550 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2551   ShouldNotReachHere();  // generic vector operands not supported
 2552   return false;
 2553 }
 2554 
 2555 // Return whether or not this register is ever used as an argument.
 2556 // This function is used on startup to build the trampoline stubs in
 2557 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2558 // call in the trampoline, and arguments in those registers not be
 2559 // available to the callee.
 2560 bool Matcher::can_be_java_arg(int reg)
 2561 {
 2562   return
 2563     reg ==  R0_num || reg == R0_H_num ||
 2564     reg ==  R1_num || reg == R1_H_num ||
 2565     reg ==  R2_num || reg == R2_H_num ||
 2566     reg ==  R3_num || reg == R3_H_num ||
 2567     reg ==  R4_num || reg == R4_H_num ||
 2568     reg ==  R5_num || reg == R5_H_num ||
 2569     reg ==  R6_num || reg == R6_H_num ||
 2570     reg ==  R7_num || reg == R7_H_num ||
 2571     reg ==  V0_num || reg == V0_H_num ||
 2572     reg ==  V1_num || reg == V1_H_num ||
 2573     reg ==  V2_num || reg == V2_H_num ||
 2574     reg ==  V3_num || reg == V3_H_num ||
 2575     reg ==  V4_num || reg == V4_H_num ||
 2576     reg ==  V5_num || reg == V5_H_num ||
 2577     reg ==  V6_num || reg == V6_H_num ||
 2578     reg ==  V7_num || reg == V7_H_num;
 2579 }
 2580 
 2581 bool Matcher::is_spillable_arg(int reg)
 2582 {
 2583   return can_be_java_arg(reg);
 2584 }
 2585 
 2586 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2587   return false;
 2588 }
 2589 
 2590 RegMask Matcher::divI_proj_mask() {
 2591   ShouldNotReachHere();
 2592   return RegMask();
 2593 }
 2594 
 2595 // Register for MODI projection of divmodI.
 2596 RegMask Matcher::modI_proj_mask() {
 2597   ShouldNotReachHere();
 2598   return RegMask();
 2599 }
 2600 
 2601 // Register for DIVL projection of divmodL.
 2602 RegMask Matcher::divL_proj_mask() {
 2603   ShouldNotReachHere();
 2604   return RegMask();
 2605 }
 2606 
 2607 // Register for MODL projection of divmodL.
 2608 RegMask Matcher::modL_proj_mask() {
 2609   ShouldNotReachHere();
 2610   return RegMask();
 2611 }
 2612 
 2613 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2614   return FP_REG_mask();
 2615 }
 2616 
 2617 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2618   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2619     Node* u = addp->fast_out(i);
 2620     if (u->is_LoadStore()) {
 2621       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2622       // instructions) only take register indirect as an operand, so
 2623       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2624       // must fail.
 2625       return false;
 2626     }
 2627     if (u->is_Mem()) {
 2628       int opsize = u->as_Mem()->memory_size();
 2629       assert(opsize > 0, "unexpected memory operand size");
 2630       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2631         return false;
 2632       }
 2633     }
 2634   }
 2635   return true;
 2636 }
 2637 
 2638 // Should the matcher clone input 'm' of node 'n'?
 2639 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2640   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2641     mstack.push(m, Visit);           // m = ShiftCntV
 2642     return true;
 2643   }
 2644   return false;
 2645 }
 2646 
 2647 // Should the Matcher clone shifts on addressing modes, expecting them
 2648 // to be subsumed into complex addressing expressions or compute them
 2649 // into registers?
 2650 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2651   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2652     return true;
 2653   }
 2654 
 2655   Node *off = m->in(AddPNode::Offset);
 2656   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2657       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2658       // Are there other uses besides address expressions?
 2659       !is_visited(off)) {
 2660     address_visited.set(off->_idx); // Flag as address_visited
 2661     mstack.push(off->in(2), Visit);
 2662     Node *conv = off->in(1);
 2663     if (conv->Opcode() == Op_ConvI2L &&
 2664         // Are there other uses besides address expressions?
 2665         !is_visited(conv)) {
 2666       address_visited.set(conv->_idx); // Flag as address_visited
 2667       mstack.push(conv->in(1), Pre_Visit);
 2668     } else {
 2669       mstack.push(conv, Pre_Visit);
 2670     }
 2671     address_visited.test_set(m->_idx); // Flag as address_visited
 2672     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2673     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2674     return true;
 2675   } else if (off->Opcode() == Op_ConvI2L &&
 2676              // Are there other uses besides address expressions?
 2677              !is_visited(off)) {
 2678     address_visited.test_set(m->_idx); // Flag as address_visited
 2679     address_visited.set(off->_idx); // Flag as address_visited
 2680     mstack.push(off->in(1), Pre_Visit);
 2681     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2682     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2683     return true;
 2684   }
 2685   return false;
 2686 }
 2687 
 2688 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2689   C2_MacroAssembler _masm(&cbuf);                                       \
 2690   {                                                                     \
 2691     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2692     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2693     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2694     __ INSN(REG, as_Register(BASE));                                    \
 2695   }
 2696 
 2697 
 2698 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2699   {
 2700     Address::extend scale;
 2701 
 2702     // Hooboy, this is fugly.  We need a way to communicate to the
 2703     // encoder that the index needs to be sign extended, so we have to
 2704     // enumerate all the cases.
 2705     switch (opcode) {
 2706     case INDINDEXSCALEDI2L:
 2707     case INDINDEXSCALEDI2LN:
 2708     case INDINDEXI2L:
 2709     case INDINDEXI2LN:
 2710       scale = Address::sxtw(size);
 2711       break;
 2712     default:
 2713       scale = Address::lsl(size);
 2714     }
 2715 
 2716     if (index == -1) {
 2717       return Address(base, disp);
 2718     } else {
 2719       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2720       return Address(base, as_Register(index), scale);
 2721     }
 2722   }
 2723 
 2724 
 2725 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2726 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2727 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2728 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2729                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2730 
 2731   // Used for all non-volatile memory accesses.  The use of
 2732   // $mem->opcode() to discover whether this pattern uses sign-extended
 2733   // offsets is something of a kludge.
 2734   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2735                         Register reg, int opcode,
 2736                         Register base, int index, int scale, int disp,
 2737                         int size_in_memory)
 2738   {
 2739     Address addr = mem2address(opcode, base, index, scale, disp);
 2740     if (addr.getMode() == Address::base_plus_offset) {
 2741       /* If we get an out-of-range offset it is a bug in the compiler,
 2742          so we assert here. */
 2743       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2744              "c2 compiler bug");
 2745       /* Fix up any out-of-range offsets. */
 2746       assert_different_registers(rscratch1, base);
 2747       assert_different_registers(rscratch1, reg);
 2748       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2749     }
 2750     (masm.*insn)(reg, addr);
 2751   }
 2752 
 2753   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2754                         FloatRegister reg, int opcode,
 2755                         Register base, int index, int size, int disp,
 2756                         int size_in_memory)
 2757   {
 2758     Address::extend scale;
 2759 
 2760     switch (opcode) {
 2761     case INDINDEXSCALEDI2L:
 2762     case INDINDEXSCALEDI2LN:
 2763       scale = Address::sxtw(size);
 2764       break;
 2765     default:
 2766       scale = Address::lsl(size);
 2767     }
 2768 
 2769     if (index == -1) {
 2770       /* If we get an out-of-range offset it is a bug in the compiler,
 2771          so we assert here. */
 2772       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2773       /* Fix up any out-of-range offsets. */
 2774       assert_different_registers(rscratch1, base);
 2775       Address addr = Address(base, disp);
 2776       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2777       (masm.*insn)(reg, addr);
 2778     } else {
 2779       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2780       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2781     }
 2782   }
 2783 
 2784   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2785                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2786                         int opcode, Register base, int index, int size, int disp)
 2787   {
 2788     if (index == -1) {
 2789       (masm.*insn)(reg, T, Address(base, disp));
 2790     } else {
 2791       assert(disp == 0, "unsupported address mode");
 2792       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2793     }
 2794   }
 2795 
 2796 %}
 2797 
 2798 
 2799 
 2800 //----------ENCODING BLOCK-----------------------------------------------------
 2801 // This block specifies the encoding classes used by the compiler to
 2802 // output byte streams.  Encoding classes are parameterized macros
 2803 // used by Machine Instruction Nodes in order to generate the bit
 2804 // encoding of the instruction.  Operands specify their base encoding
 2805 // interface with the interface keyword.  There are currently
 2806 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2807 // COND_INTER.  REG_INTER causes an operand to generate a function
 2808 // which returns its register number when queried.  CONST_INTER causes
 2809 // an operand to generate a function which returns the value of the
 2810 // constant when queried.  MEMORY_INTER causes an operand to generate
 2811 // four functions which return the Base Register, the Index Register,
 2812 // the Scale Value, and the Offset Value of the operand when queried.
 2813 // COND_INTER causes an operand to generate six functions which return
 2814 // the encoding code (ie - encoding bits for the instruction)
 2815 // associated with each basic boolean condition for a conditional
 2816 // instruction.
 2817 //
 2818 // Instructions specify two basic values for encoding.  Again, a
 2819 // function is available to check if the constant displacement is an
 2820 // oop. They use the ins_encode keyword to specify their encoding
 2821 // classes (which must be a sequence of enc_class names, and their
 2822 // parameters, specified in the encoding block), and they use the
 2823 // opcode keyword to specify, in order, their primary, secondary, and
 2824 // tertiary opcode.  Only the opcode sections which a particular
 2825 // instruction needs for encoding need to be specified.
 2826 encode %{
 2827   // Build emit functions for each basic byte or larger field in the
 2828   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2829   // from C++ code in the enc_class source block.  Emit functions will
 2830   // live in the main source block for now.  In future, we can
 2831   // generalize this by adding a syntax that specifies the sizes of
 2832   // fields in an order, so that the adlc can build the emit functions
 2833   // automagically
 2834 
 2835   // catch all for unimplemented encodings
 2836   enc_class enc_unimplemented %{
 2837     C2_MacroAssembler _masm(&cbuf);
 2838     __ unimplemented("C2 catch all");
 2839   %}
 2840 
 2841   // BEGIN Non-volatile memory access
 2842 
 2843   // This encoding class is generated automatically from ad_encode.m4.
 2844   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2845   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2846     Register dst_reg = as_Register($dst$$reg);
 2847     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2848                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2849   %}
 2850 
 2851   // This encoding class is generated automatically from ad_encode.m4.
 2852   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2853   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2854     Register dst_reg = as_Register($dst$$reg);
 2855     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2856                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2857   %}
 2858 
 2859   // This encoding class is generated automatically from ad_encode.m4.
 2860   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2861   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2862     Register dst_reg = as_Register($dst$$reg);
 2863     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2864                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2865   %}
 2866 
 2867   // This encoding class is generated automatically from ad_encode.m4.
 2868   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2869   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2870     Register dst_reg = as_Register($dst$$reg);
 2871     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2872                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2873   %}
 2874 
 2875   // This encoding class is generated automatically from ad_encode.m4.
 2876   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2877   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2878     Register dst_reg = as_Register($dst$$reg);
 2879     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2880                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2881   %}
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2942     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2950     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2958     Register src_reg = as_Register($src$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_strb0(memory1 mem) %{
 2966     C2_MacroAssembler _masm(&cbuf);
 2967     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2974     Register src_reg = as_Register($src$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_strh0(memory2 mem) %{
 2982     C2_MacroAssembler _masm(&cbuf);
 2983     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2990     Register src_reg = as_Register($src$$reg);
 2991     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strw0(memory4 mem) %{
 2998     C2_MacroAssembler _masm(&cbuf);
 2999     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3006     Register src_reg = as_Register($src$$reg);
 3007     // we sometimes get asked to store the stack pointer into the
 3008     // current thread -- we cannot do that directly on AArch64
 3009     if (src_reg == r31_sp) {
 3010       C2_MacroAssembler _masm(&cbuf);
 3011       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3012       __ mov(rscratch2, sp);
 3013       src_reg = rscratch2;
 3014     }
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_str0(memory8 mem) %{
 3022     C2_MacroAssembler _masm(&cbuf);
 3023     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3030     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3032                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3033   %}
 3034 
 3035   // This encoding class is generated automatically from ad_encode.m4.
 3036   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3037   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3038     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3039     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3046       C2_MacroAssembler _masm(&cbuf);
 3047       __ membar(Assembler::StoreStore);
 3048       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3049                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3050   %}
 3051 
 3052   // END Non-volatile memory access
 3053 
 3054   // Vector loads and stores
 3055   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3056     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3057     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3058        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3059   %}
 3060 
 3061   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3062     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3063     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3064        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3065   %}
 3066 
 3067   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3068     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3069     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3070        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3071   %}
 3072 
 3073   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3074     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3075     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3076        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3077   %}
 3078 
 3079   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3080     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3081     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3082        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3086     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3087     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3088        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3092     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3094        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3098     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3099     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3100        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3101   %}
 3102 
 3103   // volatile loads and stores
 3104 
 3105   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3106     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3107                  rscratch1, stlrb);
 3108   %}
 3109 
 3110   enc_class aarch64_enc_stlrb0(memory mem) %{
 3111     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3112                  rscratch1, stlrb);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3116     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3117                  rscratch1, stlrh);
 3118   %}
 3119 
 3120   enc_class aarch64_enc_stlrh0(memory mem) %{
 3121     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3122                  rscratch1, stlrh);
 3123   %}
 3124 
 3125   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3126     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3127                  rscratch1, stlrw);
 3128   %}
 3129 
 3130   enc_class aarch64_enc_stlrw0(memory mem) %{
 3131     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3132                  rscratch1, stlrw);
 3133   %}
 3134 
 3135   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3136     Register dst_reg = as_Register($dst$$reg);
 3137     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3138              rscratch1, ldarb);
 3139     __ sxtbw(dst_reg, dst_reg);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3143     Register dst_reg = as_Register($dst$$reg);
 3144     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3145              rscratch1, ldarb);
 3146     __ sxtb(dst_reg, dst_reg);
 3147   %}
 3148 
 3149   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3150     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3151              rscratch1, ldarb);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3155     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3156              rscratch1, ldarb);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3160     Register dst_reg = as_Register($dst$$reg);
 3161     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3162              rscratch1, ldarh);
 3163     __ sxthw(dst_reg, dst_reg);
 3164   %}
 3165 
 3166   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3167     Register dst_reg = as_Register($dst$$reg);
 3168     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3169              rscratch1, ldarh);
 3170     __ sxth(dst_reg, dst_reg);
 3171   %}
 3172 
 3173   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3174     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3175              rscratch1, ldarh);
 3176   %}
 3177 
 3178   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3179     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3180              rscratch1, ldarh);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3184     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3185              rscratch1, ldarw);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3189     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3190              rscratch1, ldarw);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3194     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldar);
 3196   %}
 3197 
 3198   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3199     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3200              rscratch1, ldarw);
 3201     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3202   %}
 3203 
 3204   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3205     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3206              rscratch1, ldar);
 3207     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3208   %}
 3209 
 3210   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3211     Register src_reg = as_Register($src$$reg);
 3212     // we sometimes get asked to store the stack pointer into the
 3213     // current thread -- we cannot do that directly on AArch64
 3214     if (src_reg == r31_sp) {
 3215       C2_MacroAssembler _masm(&cbuf);
 3216       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3217       __ mov(rscratch2, sp);
 3218       src_reg = rscratch2;
 3219     }
 3220     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3221                  rscratch1, stlr);
 3222   %}
 3223 
 3224   enc_class aarch64_enc_stlr0(memory mem) %{
 3225     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3226                  rscratch1, stlr);
 3227   %}
 3228 
 3229   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3230     {
 3231       C2_MacroAssembler _masm(&cbuf);
 3232       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3233       __ fmovs(rscratch2, src_reg);
 3234     }
 3235     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3236                  rscratch1, stlrw);
 3237   %}
 3238 
 3239   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3240     {
 3241       C2_MacroAssembler _masm(&cbuf);
 3242       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3243       __ fmovd(rscratch2, src_reg);
 3244     }
 3245     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3246                  rscratch1, stlr);
 3247   %}
 3248 
 3249   // synchronized read/update encodings
 3250 
 3251   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3252     C2_MacroAssembler _masm(&cbuf);
 3253     Register dst_reg = as_Register($dst$$reg);
 3254     Register base = as_Register($mem$$base);
 3255     int index = $mem$$index;
 3256     int scale = $mem$$scale;
 3257     int disp = $mem$$disp;
 3258     if (index == -1) {
 3259        if (disp != 0) {
 3260         __ lea(rscratch1, Address(base, disp));
 3261         __ ldaxr(dst_reg, rscratch1);
 3262       } else {
 3263         // TODO
 3264         // should we ever get anything other than this case?
 3265         __ ldaxr(dst_reg, base);
 3266       }
 3267     } else {
 3268       Register index_reg = as_Register(index);
 3269       if (disp == 0) {
 3270         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3271         __ ldaxr(dst_reg, rscratch1);
 3272       } else {
 3273         __ lea(rscratch1, Address(base, disp));
 3274         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3275         __ ldaxr(dst_reg, rscratch1);
 3276       }
 3277     }
 3278   %}
 3279 
 3280   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3281     C2_MacroAssembler _masm(&cbuf);
 3282     Register src_reg = as_Register($src$$reg);
 3283     Register base = as_Register($mem$$base);
 3284     int index = $mem$$index;
 3285     int scale = $mem$$scale;
 3286     int disp = $mem$$disp;
 3287     if (index == -1) {
 3288        if (disp != 0) {
 3289         __ lea(rscratch2, Address(base, disp));
 3290         __ stlxr(rscratch1, src_reg, rscratch2);
 3291       } else {
 3292         // TODO
 3293         // should we ever get anything other than this case?
 3294         __ stlxr(rscratch1, src_reg, base);
 3295       }
 3296     } else {
 3297       Register index_reg = as_Register(index);
 3298       if (disp == 0) {
 3299         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3300         __ stlxr(rscratch1, src_reg, rscratch2);
 3301       } else {
 3302         __ lea(rscratch2, Address(base, disp));
 3303         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3304         __ stlxr(rscratch1, src_reg, rscratch2);
 3305       }
 3306     }
 3307     __ cmpw(rscratch1, zr);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3311     C2_MacroAssembler _masm(&cbuf);
 3312     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3313     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3314                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3315                /*weak*/ false, noreg);
 3316   %}
 3317 
 3318   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3319     C2_MacroAssembler _masm(&cbuf);
 3320     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3321     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3322                Assembler::word, /*acquire*/ false, /*release*/ true,
 3323                /*weak*/ false, noreg);
 3324   %}
 3325 
 3326   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3327     C2_MacroAssembler _masm(&cbuf);
 3328     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3329     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3330                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3331                /*weak*/ false, noreg);
 3332   %}
 3333 
 3334   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3335     C2_MacroAssembler _masm(&cbuf);
 3336     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3337     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3338                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3339                /*weak*/ false, noreg);
 3340   %}
 3341 
 3342 
 3343   // The only difference between aarch64_enc_cmpxchg and
 3344   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3345   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3346   // lock.
 3347   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3348     C2_MacroAssembler _masm(&cbuf);
 3349     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3350     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3351                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3352                /*weak*/ false, noreg);
 3353   %}
 3354 
 3355   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3356     C2_MacroAssembler _masm(&cbuf);
 3357     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3358     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3359                Assembler::word, /*acquire*/ true, /*release*/ true,
 3360                /*weak*/ false, noreg);
 3361   %}
 3362 
 3363   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3364     C2_MacroAssembler _masm(&cbuf);
 3365     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3366     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3367                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3368                /*weak*/ false, noreg);
 3369   %}
 3370 
 3371   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3372     C2_MacroAssembler _masm(&cbuf);
 3373     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3374     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3375                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3376                /*weak*/ false, noreg);
 3377   %}
 3378 
 3379   // auxiliary used for CompareAndSwapX to set result register
 3380   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3381     C2_MacroAssembler _masm(&cbuf);
 3382     Register res_reg = as_Register($res$$reg);
 3383     __ cset(res_reg, Assembler::EQ);
 3384   %}
 3385 
 3386   // prefetch encodings
 3387 
 3388   enc_class aarch64_enc_prefetchw(memory mem) %{
 3389     C2_MacroAssembler _masm(&cbuf);
 3390     Register base = as_Register($mem$$base);
 3391     int index = $mem$$index;
 3392     int scale = $mem$$scale;
 3393     int disp = $mem$$disp;
 3394     if (index == -1) {
 3395       __ prfm(Address(base, disp), PSTL1KEEP);
 3396     } else {
 3397       Register index_reg = as_Register(index);
 3398       if (disp == 0) {
 3399         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3400       } else {
 3401         __ lea(rscratch1, Address(base, disp));
 3402 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3403       }
 3404     }
 3405   %}
 3406 
 3407   /// mov envcodings
 3408 
 3409   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3410     C2_MacroAssembler _masm(&cbuf);
 3411     uint32_t con = (uint32_t)$src$$constant;
 3412     Register dst_reg = as_Register($dst$$reg);
 3413     if (con == 0) {
 3414       __ movw(dst_reg, zr);
 3415     } else {
 3416       __ movw(dst_reg, con);
 3417     }
 3418   %}
 3419 
 3420   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3421     C2_MacroAssembler _masm(&cbuf);
 3422     Register dst_reg = as_Register($dst$$reg);
 3423     uint64_t con = (uint64_t)$src$$constant;
 3424     if (con == 0) {
 3425       __ mov(dst_reg, zr);
 3426     } else {
 3427       __ mov(dst_reg, con);
 3428     }
 3429   %}
 3430 
 3431   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3432     C2_MacroAssembler _masm(&cbuf);
 3433     Register dst_reg = as_Register($dst$$reg);
 3434     address con = (address)$src$$constant;
 3435     if (con == NULL || con == (address)1) {
 3436       ShouldNotReachHere();
 3437     } else {
 3438       relocInfo::relocType rtype = $src->constant_reloc();
 3439       if (rtype == relocInfo::oop_type) {
 3440         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3441       } else if (rtype == relocInfo::metadata_type) {
 3442         __ mov_metadata(dst_reg, (Metadata*)con);
 3443       } else {
 3444         assert(rtype == relocInfo::none, "unexpected reloc type");
 3445         if (! __ is_valid_AArch64_address(con) ||
 3446             con < (address)(uintptr_t)os::vm_page_size()) {
 3447           __ mov(dst_reg, con);
 3448         } else {
 3449           uint64_t offset;
 3450           __ adrp(dst_reg, con, offset);
 3451           __ add(dst_reg, dst_reg, offset);
 3452         }
 3453       }
 3454     }
 3455   %}
 3456 
 3457   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3458     C2_MacroAssembler _masm(&cbuf);
 3459     Register dst_reg = as_Register($dst$$reg);
 3460     __ mov(dst_reg, zr);
 3461   %}
 3462 
 3463   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3464     C2_MacroAssembler _masm(&cbuf);
 3465     Register dst_reg = as_Register($dst$$reg);
 3466     __ mov(dst_reg, (uint64_t)1);
 3467   %}
 3468 
 3469   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3470     C2_MacroAssembler _masm(&cbuf);
 3471     __ load_byte_map_base($dst$$Register);
 3472   %}
 3473 
 3474   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3475     C2_MacroAssembler _masm(&cbuf);
 3476     Register dst_reg = as_Register($dst$$reg);
 3477     address con = (address)$src$$constant;
 3478     if (con == NULL) {
 3479       ShouldNotReachHere();
 3480     } else {
 3481       relocInfo::relocType rtype = $src->constant_reloc();
 3482       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3483       __ set_narrow_oop(dst_reg, (jobject)con);
 3484     }
 3485   %}
 3486 
 3487   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3488     C2_MacroAssembler _masm(&cbuf);
 3489     Register dst_reg = as_Register($dst$$reg);
 3490     __ mov(dst_reg, zr);
 3491   %}
 3492 
 3493   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3494     C2_MacroAssembler _masm(&cbuf);
 3495     Register dst_reg = as_Register($dst$$reg);
 3496     address con = (address)$src$$constant;
 3497     if (con == NULL) {
 3498       ShouldNotReachHere();
 3499     } else {
 3500       relocInfo::relocType rtype = $src->constant_reloc();
 3501       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3502       __ set_narrow_klass(dst_reg, (Klass *)con);
 3503     }
 3504   %}
 3505 
 3506   // arithmetic encodings
 3507 
 3508   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3509     C2_MacroAssembler _masm(&cbuf);
 3510     Register dst_reg = as_Register($dst$$reg);
 3511     Register src_reg = as_Register($src1$$reg);
 3512     int32_t con = (int32_t)$src2$$constant;
 3513     // add has primary == 0, subtract has primary == 1
 3514     if ($primary) { con = -con; }
 3515     if (con < 0) {
 3516       __ subw(dst_reg, src_reg, -con);
 3517     } else {
 3518       __ addw(dst_reg, src_reg, con);
 3519     }
 3520   %}
 3521 
 3522   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3523     C2_MacroAssembler _masm(&cbuf);
 3524     Register dst_reg = as_Register($dst$$reg);
 3525     Register src_reg = as_Register($src1$$reg);
 3526     int32_t con = (int32_t)$src2$$constant;
 3527     // add has primary == 0, subtract has primary == 1
 3528     if ($primary) { con = -con; }
 3529     if (con < 0) {
 3530       __ sub(dst_reg, src_reg, -con);
 3531     } else {
 3532       __ add(dst_reg, src_reg, con);
 3533     }
 3534   %}
 3535 
 3536   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3537     C2_MacroAssembler _masm(&cbuf);
 3538    Register dst_reg = as_Register($dst$$reg);
 3539    Register src1_reg = as_Register($src1$$reg);
 3540    Register src2_reg = as_Register($src2$$reg);
 3541     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3542   %}
 3543 
 3544   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3545     C2_MacroAssembler _masm(&cbuf);
 3546    Register dst_reg = as_Register($dst$$reg);
 3547    Register src1_reg = as_Register($src1$$reg);
 3548    Register src2_reg = as_Register($src2$$reg);
 3549     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3550   %}
 3551 
 3552   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3553     C2_MacroAssembler _masm(&cbuf);
 3554    Register dst_reg = as_Register($dst$$reg);
 3555    Register src1_reg = as_Register($src1$$reg);
 3556    Register src2_reg = as_Register($src2$$reg);
 3557     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3558   %}
 3559 
 3560   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3561     C2_MacroAssembler _masm(&cbuf);
 3562    Register dst_reg = as_Register($dst$$reg);
 3563    Register src1_reg = as_Register($src1$$reg);
 3564    Register src2_reg = as_Register($src2$$reg);
 3565     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3566   %}
 3567 
 3568   // compare instruction encodings
 3569 
 3570   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3571     C2_MacroAssembler _masm(&cbuf);
 3572     Register reg1 = as_Register($src1$$reg);
 3573     Register reg2 = as_Register($src2$$reg);
 3574     __ cmpw(reg1, reg2);
 3575   %}
 3576 
 3577   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3578     C2_MacroAssembler _masm(&cbuf);
 3579     Register reg = as_Register($src1$$reg);
 3580     int32_t val = $src2$$constant;
 3581     if (val >= 0) {
 3582       __ subsw(zr, reg, val);
 3583     } else {
 3584       __ addsw(zr, reg, -val);
 3585     }
 3586   %}
 3587 
 3588   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3589     C2_MacroAssembler _masm(&cbuf);
 3590     Register reg1 = as_Register($src1$$reg);
 3591     uint32_t val = (uint32_t)$src2$$constant;
 3592     __ movw(rscratch1, val);
 3593     __ cmpw(reg1, rscratch1);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598     Register reg1 = as_Register($src1$$reg);
 3599     Register reg2 = as_Register($src2$$reg);
 3600     __ cmp(reg1, reg2);
 3601   %}
 3602 
 3603   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3604     C2_MacroAssembler _masm(&cbuf);
 3605     Register reg = as_Register($src1$$reg);
 3606     int64_t val = $src2$$constant;
 3607     if (val >= 0) {
 3608       __ subs(zr, reg, val);
 3609     } else if (val != -val) {
 3610       __ adds(zr, reg, -val);
 3611     } else {
 3612     // aargh, Long.MIN_VALUE is a special case
 3613       __ orr(rscratch1, zr, (uint64_t)val);
 3614       __ subs(zr, reg, rscratch1);
 3615     }
 3616   %}
 3617 
 3618   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3619     C2_MacroAssembler _masm(&cbuf);
 3620     Register reg1 = as_Register($src1$$reg);
 3621     uint64_t val = (uint64_t)$src2$$constant;
 3622     __ mov(rscratch1, val);
 3623     __ cmp(reg1, rscratch1);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3627     C2_MacroAssembler _masm(&cbuf);
 3628     Register reg1 = as_Register($src1$$reg);
 3629     Register reg2 = as_Register($src2$$reg);
 3630     __ cmp(reg1, reg2);
 3631   %}
 3632 
 3633   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3634     C2_MacroAssembler _masm(&cbuf);
 3635     Register reg1 = as_Register($src1$$reg);
 3636     Register reg2 = as_Register($src2$$reg);
 3637     __ cmpw(reg1, reg2);
 3638   %}
 3639 
 3640   enc_class aarch64_enc_testp(iRegP src) %{
 3641     C2_MacroAssembler _masm(&cbuf);
 3642     Register reg = as_Register($src$$reg);
 3643     __ cmp(reg, zr);
 3644   %}
 3645 
 3646   enc_class aarch64_enc_testn(iRegN src) %{
 3647     C2_MacroAssembler _masm(&cbuf);
 3648     Register reg = as_Register($src$$reg);
 3649     __ cmpw(reg, zr);
 3650   %}
 3651 
 3652   enc_class aarch64_enc_b(label lbl) %{
 3653     C2_MacroAssembler _masm(&cbuf);
 3654     Label *L = $lbl$$label;
 3655     __ b(*L);
 3656   %}
 3657 
 3658   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3659     C2_MacroAssembler _masm(&cbuf);
 3660     Label *L = $lbl$$label;
 3661     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3662   %}
 3663 
 3664   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3665     C2_MacroAssembler _masm(&cbuf);
 3666     Label *L = $lbl$$label;
 3667     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3668   %}
 3669 
 3670   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3671   %{
 3672      Register sub_reg = as_Register($sub$$reg);
 3673      Register super_reg = as_Register($super$$reg);
 3674      Register temp_reg = as_Register($temp$$reg);
 3675      Register result_reg = as_Register($result$$reg);
 3676 
 3677      Label miss;
 3678      C2_MacroAssembler _masm(&cbuf);
 3679      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3680                                      NULL, &miss,
 3681                                      /*set_cond_codes:*/ true);
 3682      if ($primary) {
 3683        __ mov(result_reg, zr);
 3684      }
 3685      __ bind(miss);
 3686   %}
 3687 
 3688   enc_class aarch64_enc_java_static_call(method meth) %{
 3689     C2_MacroAssembler _masm(&cbuf);
 3690 
 3691     address addr = (address)$meth$$method;
 3692     address call;
 3693     if (!_method) {
 3694       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3695       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3696       if (call == NULL) {
 3697         ciEnv::current()->record_failure("CodeCache is full");
 3698         return;
 3699       }
 3700     } else {
 3701       int method_index = resolved_method_index(cbuf);
 3702       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3703                                                   : static_call_Relocation::spec(method_index);
 3704       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3705       if (call == NULL) {
 3706         ciEnv::current()->record_failure("CodeCache is full");
 3707         return;
 3708       }
 3709       // Emit stub for static call
 3710       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3711       if (stub == NULL) {
 3712         ciEnv::current()->record_failure("CodeCache is full");
 3713         return;
 3714       }
 3715     }
 3716 
 3717     // Only non uncommon_trap calls need to reinitialize ptrue.
 3718     if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
 3719       __ reinitialize_ptrue();
 3720     }
 3721   %}
 3722 
 3723   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3724     C2_MacroAssembler _masm(&cbuf);
 3725     int method_index = resolved_method_index(cbuf);
 3726     address call = __ ic_call((address)$meth$$method, method_index);
 3727     if (call == NULL) {
 3728       ciEnv::current()->record_failure("CodeCache is full");
 3729       return;
 3730     } else if (Compile::current()->max_vector_size() >= 16) {
 3731       __ reinitialize_ptrue();
 3732     }
 3733   %}
 3734 
 3735   enc_class aarch64_enc_call_epilog() %{
 3736     C2_MacroAssembler _masm(&cbuf);
 3737     if (VerifyStackAtCalls) {
 3738       // Check that stack depth is unchanged: find majik cookie on stack
 3739       __ call_Unimplemented();
 3740     }
 3741   %}
 3742 
 3743   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3744     C2_MacroAssembler _masm(&cbuf);
 3745 
 3746     // some calls to generated routines (arraycopy code) are scheduled
 3747     // by C2 as runtime calls. if so we can call them using a br (they
 3748     // will be in a reachable segment) otherwise we have to use a blr
 3749     // which loads the absolute address into a register.
 3750     address entry = (address)$meth$$method;
 3751     CodeBlob *cb = CodeCache::find_blob(entry);
 3752     if (cb) {
 3753       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3754       if (call == NULL) {
 3755         ciEnv::current()->record_failure("CodeCache is full");
 3756         return;
 3757       }
 3758     } else {
 3759       Label retaddr;
 3760       __ adr(rscratch2, retaddr);
 3761       __ lea(rscratch1, RuntimeAddress(entry));
 3762       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3763       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3764       __ blr(rscratch1);
 3765       __ bind(retaddr);
 3766       __ add(sp, sp, 2 * wordSize);
 3767     }
 3768     if (Compile::current()->max_vector_size() >= 16) {
 3769       __ reinitialize_ptrue();
 3770     }
 3771   %}
 3772 
 3773   enc_class aarch64_enc_rethrow() %{
 3774     C2_MacroAssembler _masm(&cbuf);
 3775     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3776   %}
 3777 
 3778   enc_class aarch64_enc_ret() %{
 3779     C2_MacroAssembler _masm(&cbuf);
 3780 #ifdef ASSERT
 3781     if (Compile::current()->max_vector_size() >= 16) {
 3782       __ verify_ptrue();
 3783     }
 3784 #endif
 3785     __ ret(lr);
 3786   %}
 3787 
 3788   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3789     C2_MacroAssembler _masm(&cbuf);
 3790     Register target_reg = as_Register($jump_target$$reg);
 3791     __ br(target_reg);
 3792   %}
 3793 
 3794   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3795     C2_MacroAssembler _masm(&cbuf);
 3796     Register target_reg = as_Register($jump_target$$reg);
 3797     // exception oop should be in r0
 3798     // ret addr has been popped into lr
 3799     // callee expects it in r3
 3800     __ mov(r3, lr);
 3801     __ br(target_reg);
 3802   %}
 3803 
 3804   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3805     C2_MacroAssembler _masm(&cbuf);
 3806     Register oop = as_Register($object$$reg);
 3807     Register box = as_Register($box$$reg);
 3808     Register disp_hdr = as_Register($tmp$$reg);
 3809     Register tmp = as_Register($tmp2$$reg);
 3810     Label cont;
 3811     Label object_has_monitor;
 3812     Label cas_failed;
 3813 
 3814     assert_different_registers(oop, box, tmp, disp_hdr);
 3815 
 3816     // Load markWord from object into displaced_header.
 3817     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3818 
 3819     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3820       __ load_klass(tmp, oop);
 3821       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3822       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3823       __ br(Assembler::NE, cont);
 3824     }
 3825 
 3826     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3827       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
 3828     }
 3829 
 3830     // Check for existing monitor
 3831     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3832 
 3833     if (LockingMode == LM_MONITOR) {
 3834       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3835       __ b(cont);
 3836     } else if (LockingMode == LM_LEGACY) {
 3837       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3838       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3839 
 3840       // Initialize the box. (Must happen before we update the object mark!)
 3841       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3842 
 3843       // Compare object markWord with an unlocked value (tmp) and if
 3844       // equal exchange the stack address of our box with object markWord.
 3845       // On failure disp_hdr contains the possibly locked markWord.
 3846       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3847                  /*release*/ true, /*weak*/ false, disp_hdr);
 3848       __ br(Assembler::EQ, cont);
 3849 
 3850       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3851 
 3852       // If the compare-and-exchange succeeded, then we found an unlocked
 3853       // object, will have now locked it will continue at label cont
 3854 
 3855       __ bind(cas_failed);
 3856       // We did not see an unlocked object so try the fast recursive case.
 3857 
 3858       // Check if the owner is self by comparing the value in the
 3859       // markWord of object (disp_hdr) with the stack pointer.
 3860       __ mov(rscratch1, sp);
 3861       __ sub(disp_hdr, disp_hdr, rscratch1);
 3862       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3863       // If condition is true we are cont and hence we can store 0 as the
 3864       // displaced header in the box, which indicates that it is a recursive lock.
 3865       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3866       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3867       __ b(cont);
 3868     } else {
 3869       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3870       __ fast_lock(oop, disp_hdr, tmp, rscratch1, cont);
 3871       __ b(cont);
 3872     }
 3873 
 3874     // Handle existing monitor.
 3875     __ bind(object_has_monitor);
 3876 
 3877     // The object's monitor m is unlocked iff m->owner == NULL,
 3878     // otherwise m->owner may contain a thread or a stack address.
 3879     //
 3880     // Try to CAS m->owner from NULL to current thread.
 3881     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3882     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3883                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3884 
 3885     if (LockingMode != LM_LIGHTWEIGHT) {
 3886       // Store a non-null value into the box to avoid looking like a re-entrant
 3887       // lock. The fast-path monitor unlock code checks for
 3888       // markWord::monitor_value so use markWord::unused_mark which has the
 3889       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3890       __ mov(tmp, (address)markWord::unused_mark().value());
 3891       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3892     }
 3893     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3894 
 3895     __ cmp(rscratch1, rthread);
 3896     __ br(Assembler::NE, cont); // Check for recursive locking
 3897 
 3898     // Recursive lock case
 3899     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3900     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3901 
 3902     __ bind(cont);
 3903     // flag == EQ indicates success
 3904     // flag == NE indicates failure
 3905   %}
 3906 
 3907   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3908     C2_MacroAssembler _masm(&cbuf);
 3909     Register oop = as_Register($object$$reg);
 3910     Register box = as_Register($box$$reg);
 3911     Register disp_hdr = as_Register($tmp$$reg);
 3912     Register tmp = as_Register($tmp2$$reg);
 3913     Label cont;
 3914     Label object_has_monitor;
 3915 
 3916     assert_different_registers(oop, box, tmp, disp_hdr);
 3917 
 3918     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3919       __ biased_locking_exit(oop, tmp, cont);
 3920     }
 3921 
 3922     if (LockingMode == LM_LEGACY) {
 3923       // Find the lock address and load the displaced header from the stack.
 3924       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3925 
 3926       // If the displaced header is 0, we have a recursive unlock.
 3927       __ cmp(disp_hdr, zr);
 3928       __ br(Assembler::EQ, cont);
 3929     }
 3930 
 3931     // Handle existing monitor.
 3932     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3933     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3934 
 3935     if (LockingMode == LM_MONITOR) {
 3936       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3937       __ b(cont);
 3938     } else if (LockingMode == LM_LEGACY) {
 3939       // Check if it is still a light weight lock, this is is true if we
 3940       // see the stack address of the basicLock in the markWord of the
 3941       // object.
 3942 
 3943       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3944                  /*release*/ true, /*weak*/ false, tmp);
 3945       __ b(cont);
 3946     } else {
 3947       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3948       __ fast_unlock(oop, tmp, box, disp_hdr, cont);
 3949       __ b(cont);
 3950     }
 3951 
 3952     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3953 
 3954     // Handle existing monitor.
 3955     __ bind(object_has_monitor);
 3956     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3957     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3958 
 3959     if (LockingMode == LM_LIGHTWEIGHT) {
 3960       // If the owner is anonymous, we need to fix it -- in an outline stub.
 3961       Register tmp2 = disp_hdr;
 3962       __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3963       // We cannot use tbnz here, the target might be too far away and cannot
 3964       // be encoded.
 3965       __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
 3966       C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 3967       Compile::current()->output()->add_stub(stub);
 3968       __ br(Assembler::NE, stub->entry());
 3969       __ bind(stub->continuation());
 3970     }
 3971 
 3972     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3973 
 3974     Label notRecursive;
 3975     __ cbz(disp_hdr, notRecursive);
 3976 
 3977     // Recursive lock
 3978     __ sub(disp_hdr, disp_hdr, 1u);
 3979     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3980     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3981     __ b(cont);
 3982 
 3983     __ bind(notRecursive);
 3984     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3985     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3986     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3987     __ cmp(rscratch1, zr); // Sets flags for result
 3988     __ cbnz(rscratch1, cont);
 3989     // need a release store here
 3990     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3991     __ stlr(zr, tmp); // set unowned
 3992 
 3993     __ bind(cont);
 3994     // flag == EQ indicates success
 3995     // flag == NE indicates failure
 3996   %}
 3997 
 3998 %}
 3999 
 4000 //----------FRAME--------------------------------------------------------------
 4001 // Definition of frame structure and management information.
 4002 //
 4003 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4004 //                             |   (to get allocators register number
 4005 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4006 //  r   CALLER     |        |
 4007 //  o     |        +--------+      pad to even-align allocators stack-slot
 4008 //  w     V        |  pad0  |        numbers; owned by CALLER
 4009 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4010 //  h     ^        |   in   |  5
 4011 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4012 //  |     |        |        |  3
 4013 //  |     |        +--------+
 4014 //  V     |        | old out|      Empty on Intel, window on Sparc
 4015 //        |    old |preserve|      Must be even aligned.
 4016 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4017 //        |        |   in   |  3   area for Intel ret address
 4018 //     Owned by    |preserve|      Empty on Sparc.
 4019 //       SELF      +--------+
 4020 //        |        |  pad2  |  2   pad to align old SP
 4021 //        |        +--------+  1
 4022 //        |        | locks  |  0
 4023 //        |        +--------+----> OptoReg::stack0(), even aligned
 4024 //        |        |  pad1  | 11   pad to align new SP
 4025 //        |        +--------+
 4026 //        |        |        | 10
 4027 //        |        | spills |  9   spills
 4028 //        V        |        |  8   (pad0 slot for callee)
 4029 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4030 //        ^        |  out   |  7
 4031 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4032 //     Owned by    +--------+
 4033 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4034 //        |    new |preserve|      Must be even-aligned.
 4035 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4036 //        |        |        |
 4037 //
 4038 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4039 //         known from SELF's arguments and the Java calling convention.
 4040 //         Region 6-7 is determined per call site.
 4041 // Note 2: If the calling convention leaves holes in the incoming argument
 4042 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4043 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4044 //         incoming area, as the Java calling convention is completely under
 4045 //         the control of the AD file.  Doubles can be sorted and packed to
 4046 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4047 //         varargs C calling conventions.
 4048 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4049 //         even aligned with pad0 as needed.
 4050 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4051 //           (the latter is true on Intel but is it false on AArch64?)
 4052 //         region 6-11 is even aligned; it may be padded out more so that
 4053 //         the region from SP to FP meets the minimum stack alignment.
 4054 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4055 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4056 //         SP meets the minimum alignment.
 4057 
 4058 frame %{
 4059   // These three registers define part of the calling convention
 4060   // between compiled code and the interpreter.
 4061 
 4062   // Inline Cache Register or Method for I2C.
 4063   inline_cache_reg(R12);
 4064 
 4065   // Number of stack slots consumed by locking an object
 4066   sync_stack_slots(2);
 4067 
 4068   // Compiled code's Frame Pointer
 4069   frame_pointer(R31);
 4070 
 4071   // Interpreter stores its frame pointer in a register which is
 4072   // stored to the stack by I2CAdaptors.
 4073   // I2CAdaptors convert from interpreted java to compiled java.
 4074   interpreter_frame_pointer(R29);
 4075 
 4076   // Stack alignment requirement
 4077   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4078 
 4079   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4080   // for calls to C.  Supports the var-args backing area for register parms.
 4081   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4082 
 4083   // The after-PROLOG location of the return address.  Location of
 4084   // return address specifies a type (REG or STACK) and a number
 4085   // representing the register number (i.e. - use a register name) or
 4086   // stack slot.
 4087   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4088   // Otherwise, it is above the locks and verification slot and alignment word
 4089   // TODO this may well be correct but need to check why that - 2 is there
 4090   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4091   // which folds in the space used for monitors
 4092   return_addr(STACK - 2 +
 4093               align_up((Compile::current()->in_preserve_stack_slots() +
 4094                         Compile::current()->fixed_slots()),
 4095                        stack_alignment_in_slots()));
 4096 
 4097   // Location of compiled Java return values.  Same as C for now.
 4098   return_value
 4099   %{
 4100     // TODO do we allow ideal_reg == Op_RegN???
 4101     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4102            "only return normal values");
 4103 
 4104     static const int lo[Op_RegL + 1] = { // enum name
 4105       0,                                 // Op_Node
 4106       0,                                 // Op_Set
 4107       R0_num,                            // Op_RegN
 4108       R0_num,                            // Op_RegI
 4109       R0_num,                            // Op_RegP
 4110       V0_num,                            // Op_RegF
 4111       V0_num,                            // Op_RegD
 4112       R0_num                             // Op_RegL
 4113     };
 4114 
 4115     static const int hi[Op_RegL + 1] = { // enum name
 4116       0,                                 // Op_Node
 4117       0,                                 // Op_Set
 4118       OptoReg::Bad,                      // Op_RegN
 4119       OptoReg::Bad,                      // Op_RegI
 4120       R0_H_num,                          // Op_RegP
 4121       OptoReg::Bad,                      // Op_RegF
 4122       V0_H_num,                          // Op_RegD
 4123       R0_H_num                           // Op_RegL
 4124     };
 4125 
 4126     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4127   %}
 4128 %}
 4129 
 4130 //----------ATTRIBUTES---------------------------------------------------------
 4131 //----------Operand Attributes-------------------------------------------------
 4132 op_attrib op_cost(1);        // Required cost attribute
 4133 
 4134 //----------Instruction Attributes---------------------------------------------
 4135 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4136 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4137 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4138                                 // a non-matching short branch variant
 4139                                 // of some long branch?
 4140 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4141                                 // be a power of 2) specifies the
 4142                                 // alignment that some part of the
 4143                                 // instruction (not necessarily the
 4144                                 // start) requires.  If > 1, a
 4145                                 // compute_padding() function must be
 4146                                 // provided for the instruction
 4147 
 4148 //----------OPERANDS-----------------------------------------------------------
 4149 // Operand definitions must precede instruction definitions for correct parsing
 4150 // in the ADLC because operands constitute user defined types which are used in
 4151 // instruction definitions.
 4152 
 4153 //----------Simple Operands----------------------------------------------------
 4154 
 4155 // Integer operands 32 bit
 4156 // 32 bit immediate
 4157 operand immI()
 4158 %{
 4159   match(ConI);
 4160 
 4161   op_cost(0);
 4162   format %{ %}
 4163   interface(CONST_INTER);
 4164 %}
 4165 
 4166 // 32 bit zero
 4167 operand immI0()
 4168 %{
 4169   predicate(n->get_int() == 0);
 4170   match(ConI);
 4171 
 4172   op_cost(0);
 4173   format %{ %}
 4174   interface(CONST_INTER);
 4175 %}
 4176 
 4177 // 32 bit unit increment
 4178 operand immI_1()
 4179 %{
 4180   predicate(n->get_int() == 1);
 4181   match(ConI);
 4182 
 4183   op_cost(0);
 4184   format %{ %}
 4185   interface(CONST_INTER);
 4186 %}
 4187 
 4188 // 32 bit unit decrement
 4189 operand immI_M1()
 4190 %{
 4191   predicate(n->get_int() == -1);
 4192   match(ConI);
 4193 
 4194   op_cost(0);
 4195   format %{ %}
 4196   interface(CONST_INTER);
 4197 %}
 4198 
 4199 // Shift values for add/sub extension shift
 4200 operand immIExt()
 4201 %{
 4202   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4203   match(ConI);
 4204 
 4205   op_cost(0);
 4206   format %{ %}
 4207   interface(CONST_INTER);
 4208 %}
 4209 
 4210 operand immI_le_4()
 4211 %{
 4212   predicate(n->get_int() <= 4);
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immI_31()
 4221 %{
 4222   predicate(n->get_int() == 31);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 operand immI_2()
 4231 %{
 4232   predicate(n->get_int() == 2);
 4233   match(ConI);
 4234 
 4235   op_cost(0);
 4236   format %{ %}
 4237   interface(CONST_INTER);
 4238 %}
 4239 
 4240 operand immI_4()
 4241 %{
 4242   predicate(n->get_int() == 4);
 4243   match(ConI);
 4244 
 4245   op_cost(0);
 4246   format %{ %}
 4247   interface(CONST_INTER);
 4248 %}
 4249 
 4250 operand immI_8()
 4251 %{
 4252   predicate(n->get_int() == 8);
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 operand immI_16()
 4261 %{
 4262   predicate(n->get_int() == 16);
 4263   match(ConI);
 4264 
 4265   op_cost(0);
 4266   format %{ %}
 4267   interface(CONST_INTER);
 4268 %}
 4269 
 4270 operand immI_24()
 4271 %{
 4272   predicate(n->get_int() == 24);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 operand immI_32()
 4281 %{
 4282   predicate(n->get_int() == 32);
 4283   match(ConI);
 4284 
 4285   op_cost(0);
 4286   format %{ %}
 4287   interface(CONST_INTER);
 4288 %}
 4289 
 4290 operand immI_48()
 4291 %{
 4292   predicate(n->get_int() == 48);
 4293   match(ConI);
 4294 
 4295   op_cost(0);
 4296   format %{ %}
 4297   interface(CONST_INTER);
 4298 %}
 4299 
 4300 operand immI_56()
 4301 %{
 4302   predicate(n->get_int() == 56);
 4303   match(ConI);
 4304 
 4305   op_cost(0);
 4306   format %{ %}
 4307   interface(CONST_INTER);
 4308 %}
 4309 
 4310 operand immI_63()
 4311 %{
 4312   predicate(n->get_int() == 63);
 4313   match(ConI);
 4314 
 4315   op_cost(0);
 4316   format %{ %}
 4317   interface(CONST_INTER);
 4318 %}
 4319 
 4320 operand immI_64()
 4321 %{
 4322   predicate(n->get_int() == 64);
 4323   match(ConI);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 operand immI_255()
 4331 %{
 4332   predicate(n->get_int() == 255);
 4333   match(ConI);
 4334 
 4335   op_cost(0);
 4336   format %{ %}
 4337   interface(CONST_INTER);
 4338 %}
 4339 
 4340 operand immI_65535()
 4341 %{
 4342   predicate(n->get_int() == 65535);
 4343   match(ConI);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 operand immI_positive()
 4351 %{
 4352   predicate(n->get_int() > 0);
 4353   match(ConI);
 4354 
 4355   op_cost(0);
 4356   format %{ %}
 4357   interface(CONST_INTER);
 4358 %}
 4359 
 4360 operand immL_255()
 4361 %{
 4362   predicate(n->get_long() == 255L);
 4363   match(ConL);
 4364 
 4365   op_cost(0);
 4366   format %{ %}
 4367   interface(CONST_INTER);
 4368 %}
 4369 
 4370 operand immL_65535()
 4371 %{
 4372   predicate(n->get_long() == 65535L);
 4373   match(ConL);
 4374 
 4375   op_cost(0);
 4376   format %{ %}
 4377   interface(CONST_INTER);
 4378 %}
 4379 
 4380 operand immL_4294967295()
 4381 %{
 4382   predicate(n->get_long() == 4294967295L);
 4383   match(ConL);
 4384 
 4385   op_cost(0);
 4386   format %{ %}
 4387   interface(CONST_INTER);
 4388 %}
 4389 
 4390 operand immL_bitmask()
 4391 %{
 4392   predicate((n->get_long() != 0)
 4393             && ((n->get_long() & 0xc000000000000000l) == 0)
 4394             && is_power_of_2(n->get_long() + 1));
 4395   match(ConL);
 4396 
 4397   op_cost(0);
 4398   format %{ %}
 4399   interface(CONST_INTER);
 4400 %}
 4401 
 4402 operand immI_bitmask()
 4403 %{
 4404   predicate((n->get_int() != 0)
 4405             && ((n->get_int() & 0xc0000000) == 0)
 4406             && is_power_of_2(n->get_int() + 1));
 4407   match(ConI);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 operand immL_positive_bitmaskI()
 4415 %{
 4416   predicate((n->get_long() != 0)
 4417             && ((julong)n->get_long() < 0x80000000ULL)
 4418             && is_power_of_2(n->get_long() + 1));
 4419   match(ConL);
 4420 
 4421   op_cost(0);
 4422   format %{ %}
 4423   interface(CONST_INTER);
 4424 %}
 4425 
 4426 // Scale values for scaled offset addressing modes (up to long but not quad)
 4427 operand immIScale()
 4428 %{
 4429   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4430   match(ConI);
 4431 
 4432   op_cost(0);
 4433   format %{ %}
 4434   interface(CONST_INTER);
 4435 %}
 4436 
 4437 // 26 bit signed offset -- for pc-relative branches
 4438 operand immI26()
 4439 %{
 4440   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4441   match(ConI);
 4442 
 4443   op_cost(0);
 4444   format %{ %}
 4445   interface(CONST_INTER);
 4446 %}
 4447 
 4448 // 19 bit signed offset -- for pc-relative loads
 4449 operand immI19()
 4450 %{
 4451   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4452   match(ConI);
 4453 
 4454   op_cost(0);
 4455   format %{ %}
 4456   interface(CONST_INTER);
 4457 %}
 4458 
 4459 // 12 bit unsigned offset -- for base plus immediate loads
 4460 operand immIU12()
 4461 %{
 4462   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4463   match(ConI);
 4464 
 4465   op_cost(0);
 4466   format %{ %}
 4467   interface(CONST_INTER);
 4468 %}
 4469 
 4470 operand immLU12()
 4471 %{
 4472   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4473   match(ConL);
 4474 
 4475   op_cost(0);
 4476   format %{ %}
 4477   interface(CONST_INTER);
 4478 %}
 4479 
 4480 // Offset for scaled or unscaled immediate loads and stores
 4481 operand immIOffset()
 4482 %{
 4483   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4484   match(ConI);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immIOffset1()
 4492 %{
 4493   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 operand immIOffset2()
 4502 %{
 4503   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4504   match(ConI);
 4505 
 4506   op_cost(0);
 4507   format %{ %}
 4508   interface(CONST_INTER);
 4509 %}
 4510 
 4511 operand immIOffset4()
 4512 %{
 4513   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4514   match(ConI);
 4515 
 4516   op_cost(0);
 4517   format %{ %}
 4518   interface(CONST_INTER);
 4519 %}
 4520 
 4521 operand immIOffset8()
 4522 %{
 4523   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4524   match(ConI);
 4525 
 4526   op_cost(0);
 4527   format %{ %}
 4528   interface(CONST_INTER);
 4529 %}
 4530 
 4531 operand immIOffset16()
 4532 %{
 4533   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4534   match(ConI);
 4535 
 4536   op_cost(0);
 4537   format %{ %}
 4538   interface(CONST_INTER);
 4539 %}
 4540 
 4541 operand immLoffset()
 4542 %{
 4543   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4544   match(ConL);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 operand immLoffset1()
 4552 %{
 4553   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4554   match(ConL);
 4555 
 4556   op_cost(0);
 4557   format %{ %}
 4558   interface(CONST_INTER);
 4559 %}
 4560 
 4561 operand immLoffset2()
 4562 %{
 4563   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4564   match(ConL);
 4565 
 4566   op_cost(0);
 4567   format %{ %}
 4568   interface(CONST_INTER);
 4569 %}
 4570 
 4571 operand immLoffset4()
 4572 %{
 4573   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4574   match(ConL);
 4575 
 4576   op_cost(0);
 4577   format %{ %}
 4578   interface(CONST_INTER);
 4579 %}
 4580 
 4581 operand immLoffset8()
 4582 %{
 4583   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4584   match(ConL);
 4585 
 4586   op_cost(0);
 4587   format %{ %}
 4588   interface(CONST_INTER);
 4589 %}
 4590 
 4591 operand immLoffset16()
 4592 %{
 4593   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4594   match(ConL);
 4595 
 4596   op_cost(0);
 4597   format %{ %}
 4598   interface(CONST_INTER);
 4599 %}
 4600 
 4601 // 8 bit signed value.
 4602 operand immI8()
 4603 %{
 4604   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4605   match(ConI);
 4606 
 4607   op_cost(0);
 4608   format %{ %}
 4609   interface(CONST_INTER);
 4610 %}
 4611 
 4612 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4613 operand immI8_shift8()
 4614 %{
 4615   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4616             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4617   match(ConI);
 4618 
 4619   op_cost(0);
 4620   format %{ %}
 4621   interface(CONST_INTER);
 4622 %}
 4623 
 4624 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4625 operand immL8_shift8()
 4626 %{
 4627   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4628             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4629   match(ConL);
 4630 
 4631   op_cost(0);
 4632   format %{ %}
 4633   interface(CONST_INTER);
 4634 %}
 4635 
 4636 // 32 bit integer valid for add sub immediate
 4637 operand immIAddSub()
 4638 %{
 4639   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4640   match(ConI);
 4641   op_cost(0);
 4642   format %{ %}
 4643   interface(CONST_INTER);
 4644 %}
 4645 
 4646 // 32 bit unsigned integer valid for logical immediate
 4647 // TODO -- check this is right when e.g the mask is 0x80000000
 4648 operand immILog()
 4649 %{
 4650   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4651   match(ConI);
 4652 
 4653   op_cost(0);
 4654   format %{ %}
 4655   interface(CONST_INTER);
 4656 %}
 4657 
 4658 // Integer operands 64 bit
 4659 // 64 bit immediate
 4660 operand immL()
 4661 %{
 4662   match(ConL);
 4663 
 4664   op_cost(0);
 4665   format %{ %}
 4666   interface(CONST_INTER);
 4667 %}
 4668 
 4669 // 64 bit zero
 4670 operand immL0()
 4671 %{
 4672   predicate(n->get_long() == 0);
 4673   match(ConL);
 4674 
 4675   op_cost(0);
 4676   format %{ %}
 4677   interface(CONST_INTER);
 4678 %}
 4679 
 4680 // 64 bit unit increment
 4681 operand immL_1()
 4682 %{
 4683   predicate(n->get_long() == 1);
 4684   match(ConL);
 4685 
 4686   op_cost(0);
 4687   format %{ %}
 4688   interface(CONST_INTER);
 4689 %}
 4690 
 4691 // 64 bit unit decrement
 4692 operand immL_M1()
 4693 %{
 4694   predicate(n->get_long() == -1);
 4695   match(ConL);
 4696 
 4697   op_cost(0);
 4698   format %{ %}
 4699   interface(CONST_INTER);
 4700 %}
 4701 
 4702 // 32 bit offset of pc in thread anchor
 4703 
 4704 operand immL_pc_off()
 4705 %{
 4706   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4707                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4708   match(ConL);
 4709 
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(CONST_INTER);
 4713 %}
 4714 
 4715 // 64 bit integer valid for add sub immediate
 4716 operand immLAddSub()
 4717 %{
 4718   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4719   match(ConL);
 4720   op_cost(0);
 4721   format %{ %}
 4722   interface(CONST_INTER);
 4723 %}
 4724 
 4725 // 64 bit integer valid for logical immediate
 4726 operand immLLog()
 4727 %{
 4728   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4729   match(ConL);
 4730   op_cost(0);
 4731   format %{ %}
 4732   interface(CONST_INTER);
 4733 %}
 4734 
 4735 // Long Immediate: low 32-bit mask
 4736 operand immL_32bits()
 4737 %{
 4738   predicate(n->get_long() == 0xFFFFFFFFL);
 4739   match(ConL);
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(CONST_INTER);
 4743 %}
 4744 
 4745 // Pointer operands
 4746 // Pointer Immediate
 4747 operand immP()
 4748 %{
 4749   match(ConP);
 4750 
 4751   op_cost(0);
 4752   format %{ %}
 4753   interface(CONST_INTER);
 4754 %}
 4755 
 4756 // NULL Pointer Immediate
 4757 operand immP0()
 4758 %{
 4759   predicate(n->get_ptr() == 0);
 4760   match(ConP);
 4761 
 4762   op_cost(0);
 4763   format %{ %}
 4764   interface(CONST_INTER);
 4765 %}
 4766 
 4767 // Pointer Immediate One
 4768 // this is used in object initialization (initial object header)
 4769 operand immP_1()
 4770 %{
 4771   predicate(n->get_ptr() == 1);
 4772   match(ConP);
 4773 
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(CONST_INTER);
 4777 %}
 4778 
 4779 // Card Table Byte Map Base
 4780 operand immByteMapBase()
 4781 %{
 4782   // Get base of card map
 4783   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4784             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4785   match(ConP);
 4786 
 4787   op_cost(0);
 4788   format %{ %}
 4789   interface(CONST_INTER);
 4790 %}
 4791 
 4792 // Pointer Immediate Minus One
 4793 // this is used when we want to write the current PC to the thread anchor
 4794 operand immP_M1()
 4795 %{
 4796   predicate(n->get_ptr() == -1);
 4797   match(ConP);
 4798 
 4799   op_cost(0);
 4800   format %{ %}
 4801   interface(CONST_INTER);
 4802 %}
 4803 
 4804 // Pointer Immediate Minus Two
 4805 // this is used when we want to write the current PC to the thread anchor
 4806 operand immP_M2()
 4807 %{
 4808   predicate(n->get_ptr() == -2);
 4809   match(ConP);
 4810 
 4811   op_cost(0);
 4812   format %{ %}
 4813   interface(CONST_INTER);
 4814 %}
 4815 
 4816 // Float and Double operands
 4817 // Double Immediate
 4818 operand immD()
 4819 %{
 4820   match(ConD);
 4821   op_cost(0);
 4822   format %{ %}
 4823   interface(CONST_INTER);
 4824 %}
 4825 
 4826 // Double Immediate: +0.0d
 4827 operand immD0()
 4828 %{
 4829   predicate(jlong_cast(n->getd()) == 0);
 4830   match(ConD);
 4831 
 4832   op_cost(0);
 4833   format %{ %}
 4834   interface(CONST_INTER);
 4835 %}
 4836 
 4837 // constant 'double +0.0'.
 4838 operand immDPacked()
 4839 %{
 4840   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4841   match(ConD);
 4842   op_cost(0);
 4843   format %{ %}
 4844   interface(CONST_INTER);
 4845 %}
 4846 
 4847 // Float Immediate
 4848 operand immF()
 4849 %{
 4850   match(ConF);
 4851   op_cost(0);
 4852   format %{ %}
 4853   interface(CONST_INTER);
 4854 %}
 4855 
 4856 // Float Immediate: +0.0f.
 4857 operand immF0()
 4858 %{
 4859   predicate(jint_cast(n->getf()) == 0);
 4860   match(ConF);
 4861 
 4862   op_cost(0);
 4863   format %{ %}
 4864   interface(CONST_INTER);
 4865 %}
 4866 
 4867 //
 4868 operand immFPacked()
 4869 %{
 4870   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4871   match(ConF);
 4872   op_cost(0);
 4873   format %{ %}
 4874   interface(CONST_INTER);
 4875 %}
 4876 
 4877 // Narrow pointer operands
 4878 // Narrow Pointer Immediate
 4879 operand immN()
 4880 %{
 4881   match(ConN);
 4882 
 4883   op_cost(0);
 4884   format %{ %}
 4885   interface(CONST_INTER);
 4886 %}
 4887 
 4888 // Narrow NULL Pointer Immediate
 4889 operand immN0()
 4890 %{
 4891   predicate(n->get_narrowcon() == 0);
 4892   match(ConN);
 4893 
 4894   op_cost(0);
 4895   format %{ %}
 4896   interface(CONST_INTER);
 4897 %}
 4898 
 4899 operand immNKlass()
 4900 %{
 4901   match(ConNKlass);
 4902 
 4903   op_cost(0);
 4904   format %{ %}
 4905   interface(CONST_INTER);
 4906 %}
 4907 
 4908 // Integer 32 bit Register Operands
 4909 // Integer 32 bitRegister (excludes SP)
 4910 operand iRegI()
 4911 %{
 4912   constraint(ALLOC_IN_RC(any_reg32));
 4913   match(RegI);
 4914   match(iRegINoSp);
 4915   op_cost(0);
 4916   format %{ %}
 4917   interface(REG_INTER);
 4918 %}
 4919 
 4920 // Integer 32 bit Register not Special
 4921 operand iRegINoSp()
 4922 %{
 4923   constraint(ALLOC_IN_RC(no_special_reg32));
 4924   match(RegI);
 4925   op_cost(0);
 4926   format %{ %}
 4927   interface(REG_INTER);
 4928 %}
 4929 
 4930 // Integer 64 bit Register Operands
 4931 // Integer 64 bit Register (includes SP)
 4932 operand iRegL()
 4933 %{
 4934   constraint(ALLOC_IN_RC(any_reg));
 4935   match(RegL);
 4936   match(iRegLNoSp);
 4937   op_cost(0);
 4938   format %{ %}
 4939   interface(REG_INTER);
 4940 %}
 4941 
 4942 // Integer 64 bit Register not Special
 4943 operand iRegLNoSp()
 4944 %{
 4945   constraint(ALLOC_IN_RC(no_special_reg));
 4946   match(RegL);
 4947   match(iRegL_R0);
 4948   format %{ %}
 4949   interface(REG_INTER);
 4950 %}
 4951 
 4952 // Pointer Register Operands
 4953 // Pointer Register
 4954 operand iRegP()
 4955 %{
 4956   constraint(ALLOC_IN_RC(ptr_reg));
 4957   match(RegP);
 4958   match(iRegPNoSp);
 4959   match(iRegP_R0);
 4960   //match(iRegP_R2);
 4961   //match(iRegP_R4);
 4962   match(iRegP_R5);
 4963   match(thread_RegP);
 4964   op_cost(0);
 4965   format %{ %}
 4966   interface(REG_INTER);
 4967 %}
 4968 
 4969 // Pointer 64 bit Register not Special
 4970 operand iRegPNoSp()
 4971 %{
 4972   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4973   match(RegP);
 4974   // match(iRegP);
 4975   // match(iRegP_R0);
 4976   // match(iRegP_R2);
 4977   // match(iRegP_R4);
 4978   // match(iRegP_R5);
 4979   // match(thread_RegP);
 4980   op_cost(0);
 4981   format %{ %}
 4982   interface(REG_INTER);
 4983 %}
 4984 
 4985 // Pointer 64 bit Register R0 only
 4986 operand iRegP_R0()
 4987 %{
 4988   constraint(ALLOC_IN_RC(r0_reg));
 4989   match(RegP);
 4990   // match(iRegP);
 4991   match(iRegPNoSp);
 4992   op_cost(0);
 4993   format %{ %}
 4994   interface(REG_INTER);
 4995 %}
 4996 
 4997 // Pointer 64 bit Register R1 only
 4998 operand iRegP_R1()
 4999 %{
 5000   constraint(ALLOC_IN_RC(r1_reg));
 5001   match(RegP);
 5002   // match(iRegP);
 5003   match(iRegPNoSp);
 5004   op_cost(0);
 5005   format %{ %}
 5006   interface(REG_INTER);
 5007 %}
 5008 
 5009 // Pointer 64 bit Register R2 only
 5010 operand iRegP_R2()
 5011 %{
 5012   constraint(ALLOC_IN_RC(r2_reg));
 5013   match(RegP);
 5014   // match(iRegP);
 5015   match(iRegPNoSp);
 5016   op_cost(0);
 5017   format %{ %}
 5018   interface(REG_INTER);
 5019 %}
 5020 
 5021 // Pointer 64 bit Register R3 only
 5022 operand iRegP_R3()
 5023 %{
 5024   constraint(ALLOC_IN_RC(r3_reg));
 5025   match(RegP);
 5026   // match(iRegP);
 5027   match(iRegPNoSp);
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(REG_INTER);
 5031 %}
 5032 
 5033 // Pointer 64 bit Register R4 only
 5034 operand iRegP_R4()
 5035 %{
 5036   constraint(ALLOC_IN_RC(r4_reg));
 5037   match(RegP);
 5038   // match(iRegP);
 5039   match(iRegPNoSp);
 5040   op_cost(0);
 5041   format %{ %}
 5042   interface(REG_INTER);
 5043 %}
 5044 
 5045 // Pointer 64 bit Register R5 only
 5046 operand iRegP_R5()
 5047 %{
 5048   constraint(ALLOC_IN_RC(r5_reg));
 5049   match(RegP);
 5050   // match(iRegP);
 5051   match(iRegPNoSp);
 5052   op_cost(0);
 5053   format %{ %}
 5054   interface(REG_INTER);
 5055 %}
 5056 
 5057 // Pointer 64 bit Register R10 only
 5058 operand iRegP_R10()
 5059 %{
 5060   constraint(ALLOC_IN_RC(r10_reg));
 5061   match(RegP);
 5062   // match(iRegP);
 5063   match(iRegPNoSp);
 5064   op_cost(0);
 5065   format %{ %}
 5066   interface(REG_INTER);
 5067 %}
 5068 
 5069 // Long 64 bit Register R0 only
 5070 operand iRegL_R0()
 5071 %{
 5072   constraint(ALLOC_IN_RC(r0_reg));
 5073   match(RegL);
 5074   match(iRegLNoSp);
 5075   op_cost(0);
 5076   format %{ %}
 5077   interface(REG_INTER);
 5078 %}
 5079 
 5080 // Long 64 bit Register R2 only
 5081 operand iRegL_R2()
 5082 %{
 5083   constraint(ALLOC_IN_RC(r2_reg));
 5084   match(RegL);
 5085   match(iRegLNoSp);
 5086   op_cost(0);
 5087   format %{ %}
 5088   interface(REG_INTER);
 5089 %}
 5090 
 5091 // Long 64 bit Register R3 only
 5092 operand iRegL_R3()
 5093 %{
 5094   constraint(ALLOC_IN_RC(r3_reg));
 5095   match(RegL);
 5096   match(iRegLNoSp);
 5097   op_cost(0);
 5098   format %{ %}
 5099   interface(REG_INTER);
 5100 %}
 5101 
 5102 // Long 64 bit Register R11 only
 5103 operand iRegL_R11()
 5104 %{
 5105   constraint(ALLOC_IN_RC(r11_reg));
 5106   match(RegL);
 5107   match(iRegLNoSp);
 5108   op_cost(0);
 5109   format %{ %}
 5110   interface(REG_INTER);
 5111 %}
 5112 
 5113 // Pointer 64 bit Register FP only
 5114 operand iRegP_FP()
 5115 %{
 5116   constraint(ALLOC_IN_RC(fp_reg));
 5117   match(RegP);
 5118   // match(iRegP);
 5119   op_cost(0);
 5120   format %{ %}
 5121   interface(REG_INTER);
 5122 %}
 5123 
 5124 // Register R0 only
 5125 operand iRegI_R0()
 5126 %{
 5127   constraint(ALLOC_IN_RC(int_r0_reg));
 5128   match(RegI);
 5129   match(iRegINoSp);
 5130   op_cost(0);
 5131   format %{ %}
 5132   interface(REG_INTER);
 5133 %}
 5134 
 5135 // Register R2 only
 5136 operand iRegI_R2()
 5137 %{
 5138   constraint(ALLOC_IN_RC(int_r2_reg));
 5139   match(RegI);
 5140   match(iRegINoSp);
 5141   op_cost(0);
 5142   format %{ %}
 5143   interface(REG_INTER);
 5144 %}
 5145 
 5146 // Register R3 only
 5147 operand iRegI_R3()
 5148 %{
 5149   constraint(ALLOC_IN_RC(int_r3_reg));
 5150   match(RegI);
 5151   match(iRegINoSp);
 5152   op_cost(0);
 5153   format %{ %}
 5154   interface(REG_INTER);
 5155 %}
 5156 
 5157 
 5158 // Register R4 only
 5159 operand iRegI_R4()
 5160 %{
 5161   constraint(ALLOC_IN_RC(int_r4_reg));
 5162   match(RegI);
 5163   match(iRegINoSp);
 5164   op_cost(0);
 5165   format %{ %}
 5166   interface(REG_INTER);
 5167 %}
 5168 
 5169 
 5170 // Pointer Register Operands
 5171 // Narrow Pointer Register
 5172 operand iRegN()
 5173 %{
 5174   constraint(ALLOC_IN_RC(any_reg32));
 5175   match(RegN);
 5176   match(iRegNNoSp);
 5177   op_cost(0);
 5178   format %{ %}
 5179   interface(REG_INTER);
 5180 %}
 5181 
 5182 operand iRegN_R0()
 5183 %{
 5184   constraint(ALLOC_IN_RC(r0_reg));
 5185   match(iRegN);
 5186   op_cost(0);
 5187   format %{ %}
 5188   interface(REG_INTER);
 5189 %}
 5190 
 5191 operand iRegN_R2()
 5192 %{
 5193   constraint(ALLOC_IN_RC(r2_reg));
 5194   match(iRegN);
 5195   op_cost(0);
 5196   format %{ %}
 5197   interface(REG_INTER);
 5198 %}
 5199 
 5200 operand iRegN_R3()
 5201 %{
 5202   constraint(ALLOC_IN_RC(r3_reg));
 5203   match(iRegN);
 5204   op_cost(0);
 5205   format %{ %}
 5206   interface(REG_INTER);
 5207 %}
 5208 
 5209 // Integer 64 bit Register not Special
 5210 operand iRegNNoSp()
 5211 %{
 5212   constraint(ALLOC_IN_RC(no_special_reg32));
 5213   match(RegN);
 5214   op_cost(0);
 5215   format %{ %}
 5216   interface(REG_INTER);
 5217 %}
 5218 
 5219 // heap base register -- used for encoding immN0
 5220 
 5221 operand iRegIHeapbase()
 5222 %{
 5223   constraint(ALLOC_IN_RC(heapbase_reg));
 5224   match(RegI);
 5225   op_cost(0);
 5226   format %{ %}
 5227   interface(REG_INTER);
 5228 %}
 5229 
 5230 // Float Register
 5231 // Float register operands
 5232 operand vRegF()
 5233 %{
 5234   constraint(ALLOC_IN_RC(float_reg));
 5235   match(RegF);
 5236 
 5237   op_cost(0);
 5238   format %{ %}
 5239   interface(REG_INTER);
 5240 %}
 5241 
 5242 // Double Register
 5243 // Double register operands
 5244 operand vRegD()
 5245 %{
 5246   constraint(ALLOC_IN_RC(double_reg));
 5247   match(RegD);
 5248 
 5249   op_cost(0);
 5250   format %{ %}
 5251   interface(REG_INTER);
 5252 %}
 5253 
 5254 // Generic vector class. This will be used for
 5255 // all vector operands, including NEON and SVE,
 5256 // but currently only used for SVE VecA.
 5257 operand vReg()
 5258 %{
 5259   constraint(ALLOC_IN_RC(vectora_reg));
 5260   match(VecA);
 5261   op_cost(0);
 5262   format %{ %}
 5263   interface(REG_INTER);
 5264 %}
 5265 
 5266 operand vecD()
 5267 %{
 5268   constraint(ALLOC_IN_RC(vectord_reg));
 5269   match(VecD);
 5270 
 5271   op_cost(0);
 5272   format %{ %}
 5273   interface(REG_INTER);
 5274 %}
 5275 
 5276 operand vecX()
 5277 %{
 5278   constraint(ALLOC_IN_RC(vectorx_reg));
 5279   match(VecX);
 5280 
 5281   op_cost(0);
 5282   format %{ %}
 5283   interface(REG_INTER);
 5284 %}
 5285 
 5286 operand vRegD_V0()
 5287 %{
 5288   constraint(ALLOC_IN_RC(v0_reg));
 5289   match(RegD);
 5290   op_cost(0);
 5291   format %{ %}
 5292   interface(REG_INTER);
 5293 %}
 5294 
 5295 operand vRegD_V1()
 5296 %{
 5297   constraint(ALLOC_IN_RC(v1_reg));
 5298   match(RegD);
 5299   op_cost(0);
 5300   format %{ %}
 5301   interface(REG_INTER);
 5302 %}
 5303 
 5304 operand vRegD_V2()
 5305 %{
 5306   constraint(ALLOC_IN_RC(v2_reg));
 5307   match(RegD);
 5308   op_cost(0);
 5309   format %{ %}
 5310   interface(REG_INTER);
 5311 %}
 5312 
 5313 operand vRegD_V3()
 5314 %{
 5315   constraint(ALLOC_IN_RC(v3_reg));
 5316   match(RegD);
 5317   op_cost(0);
 5318   format %{ %}
 5319   interface(REG_INTER);
 5320 %}
 5321 
 5322 operand vRegD_V4()
 5323 %{
 5324   constraint(ALLOC_IN_RC(v4_reg));
 5325   match(RegD);
 5326   op_cost(0);
 5327   format %{ %}
 5328   interface(REG_INTER);
 5329 %}
 5330 
 5331 operand vRegD_V5()
 5332 %{
 5333   constraint(ALLOC_IN_RC(v5_reg));
 5334   match(RegD);
 5335   op_cost(0);
 5336   format %{ %}
 5337   interface(REG_INTER);
 5338 %}
 5339 
 5340 operand vRegD_V6()
 5341 %{
 5342   constraint(ALLOC_IN_RC(v6_reg));
 5343   match(RegD);
 5344   op_cost(0);
 5345   format %{ %}
 5346   interface(REG_INTER);
 5347 %}
 5348 
 5349 operand vRegD_V7()
 5350 %{
 5351   constraint(ALLOC_IN_RC(v7_reg));
 5352   match(RegD);
 5353   op_cost(0);
 5354   format %{ %}
 5355   interface(REG_INTER);
 5356 %}
 5357 
 5358 operand vRegD_V8()
 5359 %{
 5360   constraint(ALLOC_IN_RC(v8_reg));
 5361   match(RegD);
 5362   op_cost(0);
 5363   format %{ %}
 5364   interface(REG_INTER);
 5365 %}
 5366 
 5367 operand vRegD_V9()
 5368 %{
 5369   constraint(ALLOC_IN_RC(v9_reg));
 5370   match(RegD);
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vRegD_V10()
 5377 %{
 5378   constraint(ALLOC_IN_RC(v10_reg));
 5379   match(RegD);
 5380   op_cost(0);
 5381   format %{ %}
 5382   interface(REG_INTER);
 5383 %}
 5384 
 5385 operand vRegD_V11()
 5386 %{
 5387   constraint(ALLOC_IN_RC(v11_reg));
 5388   match(RegD);
 5389   op_cost(0);
 5390   format %{ %}
 5391   interface(REG_INTER);
 5392 %}
 5393 
 5394 operand vRegD_V12()
 5395 %{
 5396   constraint(ALLOC_IN_RC(v12_reg));
 5397   match(RegD);
 5398   op_cost(0);
 5399   format %{ %}
 5400   interface(REG_INTER);
 5401 %}
 5402 
 5403 operand vRegD_V13()
 5404 %{
 5405   constraint(ALLOC_IN_RC(v13_reg));
 5406   match(RegD);
 5407   op_cost(0);
 5408   format %{ %}
 5409   interface(REG_INTER);
 5410 %}
 5411 
 5412 operand vRegD_V14()
 5413 %{
 5414   constraint(ALLOC_IN_RC(v14_reg));
 5415   match(RegD);
 5416   op_cost(0);
 5417   format %{ %}
 5418   interface(REG_INTER);
 5419 %}
 5420 
 5421 operand vRegD_V15()
 5422 %{
 5423   constraint(ALLOC_IN_RC(v15_reg));
 5424   match(RegD);
 5425   op_cost(0);
 5426   format %{ %}
 5427   interface(REG_INTER);
 5428 %}
 5429 
 5430 operand vRegD_V16()
 5431 %{
 5432   constraint(ALLOC_IN_RC(v16_reg));
 5433   match(RegD);
 5434   op_cost(0);
 5435   format %{ %}
 5436   interface(REG_INTER);
 5437 %}
 5438 
 5439 operand vRegD_V17()
 5440 %{
 5441   constraint(ALLOC_IN_RC(v17_reg));
 5442   match(RegD);
 5443   op_cost(0);
 5444   format %{ %}
 5445   interface(REG_INTER);
 5446 %}
 5447 
 5448 operand vRegD_V18()
 5449 %{
 5450   constraint(ALLOC_IN_RC(v18_reg));
 5451   match(RegD);
 5452   op_cost(0);
 5453   format %{ %}
 5454   interface(REG_INTER);
 5455 %}
 5456 
 5457 operand vRegD_V19()
 5458 %{
 5459   constraint(ALLOC_IN_RC(v19_reg));
 5460   match(RegD);
 5461   op_cost(0);
 5462   format %{ %}
 5463   interface(REG_INTER);
 5464 %}
 5465 
 5466 operand vRegD_V20()
 5467 %{
 5468   constraint(ALLOC_IN_RC(v20_reg));
 5469   match(RegD);
 5470   op_cost(0);
 5471   format %{ %}
 5472   interface(REG_INTER);
 5473 %}
 5474 
 5475 operand vRegD_V21()
 5476 %{
 5477   constraint(ALLOC_IN_RC(v21_reg));
 5478   match(RegD);
 5479   op_cost(0);
 5480   format %{ %}
 5481   interface(REG_INTER);
 5482 %}
 5483 
 5484 operand vRegD_V22()
 5485 %{
 5486   constraint(ALLOC_IN_RC(v22_reg));
 5487   match(RegD);
 5488   op_cost(0);
 5489   format %{ %}
 5490   interface(REG_INTER);
 5491 %}
 5492 
 5493 operand vRegD_V23()
 5494 %{
 5495   constraint(ALLOC_IN_RC(v23_reg));
 5496   match(RegD);
 5497   op_cost(0);
 5498   format %{ %}
 5499   interface(REG_INTER);
 5500 %}
 5501 
 5502 operand vRegD_V24()
 5503 %{
 5504   constraint(ALLOC_IN_RC(v24_reg));
 5505   match(RegD);
 5506   op_cost(0);
 5507   format %{ %}
 5508   interface(REG_INTER);
 5509 %}
 5510 
 5511 operand vRegD_V25()
 5512 %{
 5513   constraint(ALLOC_IN_RC(v25_reg));
 5514   match(RegD);
 5515   op_cost(0);
 5516   format %{ %}
 5517   interface(REG_INTER);
 5518 %}
 5519 
 5520 operand vRegD_V26()
 5521 %{
 5522   constraint(ALLOC_IN_RC(v26_reg));
 5523   match(RegD);
 5524   op_cost(0);
 5525   format %{ %}
 5526   interface(REG_INTER);
 5527 %}
 5528 
 5529 operand vRegD_V27()
 5530 %{
 5531   constraint(ALLOC_IN_RC(v27_reg));
 5532   match(RegD);
 5533   op_cost(0);
 5534   format %{ %}
 5535   interface(REG_INTER);
 5536 %}
 5537 
 5538 operand vRegD_V28()
 5539 %{
 5540   constraint(ALLOC_IN_RC(v28_reg));
 5541   match(RegD);
 5542   op_cost(0);
 5543   format %{ %}
 5544   interface(REG_INTER);
 5545 %}
 5546 
 5547 operand vRegD_V29()
 5548 %{
 5549   constraint(ALLOC_IN_RC(v29_reg));
 5550   match(RegD);
 5551   op_cost(0);
 5552   format %{ %}
 5553   interface(REG_INTER);
 5554 %}
 5555 
 5556 operand vRegD_V30()
 5557 %{
 5558   constraint(ALLOC_IN_RC(v30_reg));
 5559   match(RegD);
 5560   op_cost(0);
 5561   format %{ %}
 5562   interface(REG_INTER);
 5563 %}
 5564 
 5565 operand vRegD_V31()
 5566 %{
 5567   constraint(ALLOC_IN_RC(v31_reg));
 5568   match(RegD);
 5569   op_cost(0);
 5570   format %{ %}
 5571   interface(REG_INTER);
 5572 %}
 5573 
 5574 operand pRegGov()
 5575 %{
 5576   constraint(ALLOC_IN_RC(gov_pr));
 5577   match(RegVectMask);
 5578   op_cost(0);
 5579   format %{ %}
 5580   interface(REG_INTER);
 5581 %}
 5582 
 5583 // Flags register, used as output of signed compare instructions
 5584 
 5585 // note that on AArch64 we also use this register as the output for
 5586 // for floating point compare instructions (CmpF CmpD). this ensures
 5587 // that ordered inequality tests use GT, GE, LT or LE none of which
 5588 // pass through cases where the result is unordered i.e. one or both
 5589 // inputs to the compare is a NaN. this means that the ideal code can
 5590 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5591 // (where the comparison should always fail). EQ and NE tests are
 5592 // always generated in ideal code so that unordered folds into the NE
 5593 // case, matching the behaviour of AArch64 NE.
 5594 //
 5595 // This differs from x86 where the outputs of FP compares use a
 5596 // special FP flags registers and where compares based on this
 5597 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5598 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5599 // to explicitly handle the unordered case in branches. x86 also has
 5600 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5601 
 5602 operand rFlagsReg()
 5603 %{
 5604   constraint(ALLOC_IN_RC(int_flags));
 5605   match(RegFlags);
 5606 
 5607   op_cost(0);
 5608   format %{ "RFLAGS" %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 // Flags register, used as output of unsigned compare instructions
 5613 operand rFlagsRegU()
 5614 %{
 5615   constraint(ALLOC_IN_RC(int_flags));
 5616   match(RegFlags);
 5617 
 5618   op_cost(0);
 5619   format %{ "RFLAGSU" %}
 5620   interface(REG_INTER);
 5621 %}
 5622 
 5623 // Special Registers
 5624 
 5625 // Method Register
 5626 operand inline_cache_RegP(iRegP reg)
 5627 %{
 5628   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5629   match(reg);
 5630   match(iRegPNoSp);
 5631   op_cost(0);
 5632   format %{ %}
 5633   interface(REG_INTER);
 5634 %}
 5635 
 5636 // Thread Register
 5637 operand thread_RegP(iRegP reg)
 5638 %{
 5639   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5640   match(reg);
 5641   op_cost(0);
 5642   format %{ %}
 5643   interface(REG_INTER);
 5644 %}
 5645 
 5646 operand lr_RegP(iRegP reg)
 5647 %{
 5648   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5649   match(reg);
 5650   op_cost(0);
 5651   format %{ %}
 5652   interface(REG_INTER);
 5653 %}
 5654 
 5655 //----------Memory Operands----------------------------------------------------
 5656 
 5657 operand indirect(iRegP reg)
 5658 %{
 5659   constraint(ALLOC_IN_RC(ptr_reg));
 5660   match(reg);
 5661   op_cost(0);
 5662   format %{ "[$reg]" %}
 5663   interface(MEMORY_INTER) %{
 5664     base($reg);
 5665     index(0xffffffff);
 5666     scale(0x0);
 5667     disp(0x0);
 5668   %}
 5669 %}
 5670 
 5671 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5672 %{
 5673   constraint(ALLOC_IN_RC(ptr_reg));
 5674   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5675   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5676   op_cost(0);
 5677   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5678   interface(MEMORY_INTER) %{
 5679     base($reg);
 5680     index($ireg);
 5681     scale($scale);
 5682     disp(0x0);
 5683   %}
 5684 %}
 5685 
 5686 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5687 %{
 5688   constraint(ALLOC_IN_RC(ptr_reg));
 5689   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5690   match(AddP reg (LShiftL lreg scale));
 5691   op_cost(0);
 5692   format %{ "$reg, $lreg lsl($scale)" %}
 5693   interface(MEMORY_INTER) %{
 5694     base($reg);
 5695     index($lreg);
 5696     scale($scale);
 5697     disp(0x0);
 5698   %}
 5699 %}
 5700 
 5701 operand indIndexI2L(iRegP reg, iRegI ireg)
 5702 %{
 5703   constraint(ALLOC_IN_RC(ptr_reg));
 5704   match(AddP reg (ConvI2L ireg));
 5705   op_cost(0);
 5706   format %{ "$reg, $ireg, 0, I2L" %}
 5707   interface(MEMORY_INTER) %{
 5708     base($reg);
 5709     index($ireg);
 5710     scale(0x0);
 5711     disp(0x0);
 5712   %}
 5713 %}
 5714 
 5715 operand indIndex(iRegP reg, iRegL lreg)
 5716 %{
 5717   constraint(ALLOC_IN_RC(ptr_reg));
 5718   match(AddP reg lreg);
 5719   op_cost(0);
 5720   format %{ "$reg, $lreg" %}
 5721   interface(MEMORY_INTER) %{
 5722     base($reg);
 5723     index($lreg);
 5724     scale(0x0);
 5725     disp(0x0);
 5726   %}
 5727 %}
 5728 
 5729 operand indOffI(iRegP reg, immIOffset off)
 5730 %{
 5731   constraint(ALLOC_IN_RC(ptr_reg));
 5732   match(AddP reg off);
 5733   op_cost(0);
 5734   format %{ "[$reg, $off]" %}
 5735   interface(MEMORY_INTER) %{
 5736     base($reg);
 5737     index(0xffffffff);
 5738     scale(0x0);
 5739     disp($off);
 5740   %}
 5741 %}
 5742 
 5743 operand indOffI1(iRegP reg, immIOffset1 off)
 5744 %{
 5745   constraint(ALLOC_IN_RC(ptr_reg));
 5746   match(AddP reg off);
 5747   op_cost(0);
 5748   format %{ "[$reg, $off]" %}
 5749   interface(MEMORY_INTER) %{
 5750     base($reg);
 5751     index(0xffffffff);
 5752     scale(0x0);
 5753     disp($off);
 5754   %}
 5755 %}
 5756 
 5757 operand indOffI2(iRegP reg, immIOffset2 off)
 5758 %{
 5759   constraint(ALLOC_IN_RC(ptr_reg));
 5760   match(AddP reg off);
 5761   op_cost(0);
 5762   format %{ "[$reg, $off]" %}
 5763   interface(MEMORY_INTER) %{
 5764     base($reg);
 5765     index(0xffffffff);
 5766     scale(0x0);
 5767     disp($off);
 5768   %}
 5769 %}
 5770 
 5771 operand indOffI4(iRegP reg, immIOffset4 off)
 5772 %{
 5773   constraint(ALLOC_IN_RC(ptr_reg));
 5774   match(AddP reg off);
 5775   op_cost(0);
 5776   format %{ "[$reg, $off]" %}
 5777   interface(MEMORY_INTER) %{
 5778     base($reg);
 5779     index(0xffffffff);
 5780     scale(0x0);
 5781     disp($off);
 5782   %}
 5783 %}
 5784 
 5785 operand indOffI8(iRegP reg, immIOffset8 off)
 5786 %{
 5787   constraint(ALLOC_IN_RC(ptr_reg));
 5788   match(AddP reg off);
 5789   op_cost(0);
 5790   format %{ "[$reg, $off]" %}
 5791   interface(MEMORY_INTER) %{
 5792     base($reg);
 5793     index(0xffffffff);
 5794     scale(0x0);
 5795     disp($off);
 5796   %}
 5797 %}
 5798 
 5799 operand indOffI16(iRegP reg, immIOffset16 off)
 5800 %{
 5801   constraint(ALLOC_IN_RC(ptr_reg));
 5802   match(AddP reg off);
 5803   op_cost(0);
 5804   format %{ "[$reg, $off]" %}
 5805   interface(MEMORY_INTER) %{
 5806     base($reg);
 5807     index(0xffffffff);
 5808     scale(0x0);
 5809     disp($off);
 5810   %}
 5811 %}
 5812 
 5813 operand indOffL(iRegP reg, immLoffset off)
 5814 %{
 5815   constraint(ALLOC_IN_RC(ptr_reg));
 5816   match(AddP reg off);
 5817   op_cost(0);
 5818   format %{ "[$reg, $off]" %}
 5819   interface(MEMORY_INTER) %{
 5820     base($reg);
 5821     index(0xffffffff);
 5822     scale(0x0);
 5823     disp($off);
 5824   %}
 5825 %}
 5826 
 5827 operand indOffL1(iRegP reg, immLoffset1 off)
 5828 %{
 5829   constraint(ALLOC_IN_RC(ptr_reg));
 5830   match(AddP reg off);
 5831   op_cost(0);
 5832   format %{ "[$reg, $off]" %}
 5833   interface(MEMORY_INTER) %{
 5834     base($reg);
 5835     index(0xffffffff);
 5836     scale(0x0);
 5837     disp($off);
 5838   %}
 5839 %}
 5840 
 5841 operand indOffL2(iRegP reg, immLoffset2 off)
 5842 %{
 5843   constraint(ALLOC_IN_RC(ptr_reg));
 5844   match(AddP reg off);
 5845   op_cost(0);
 5846   format %{ "[$reg, $off]" %}
 5847   interface(MEMORY_INTER) %{
 5848     base($reg);
 5849     index(0xffffffff);
 5850     scale(0x0);
 5851     disp($off);
 5852   %}
 5853 %}
 5854 
 5855 operand indOffL4(iRegP reg, immLoffset4 off)
 5856 %{
 5857   constraint(ALLOC_IN_RC(ptr_reg));
 5858   match(AddP reg off);
 5859   op_cost(0);
 5860   format %{ "[$reg, $off]" %}
 5861   interface(MEMORY_INTER) %{
 5862     base($reg);
 5863     index(0xffffffff);
 5864     scale(0x0);
 5865     disp($off);
 5866   %}
 5867 %}
 5868 
 5869 operand indOffL8(iRegP reg, immLoffset8 off)
 5870 %{
 5871   constraint(ALLOC_IN_RC(ptr_reg));
 5872   match(AddP reg off);
 5873   op_cost(0);
 5874   format %{ "[$reg, $off]" %}
 5875   interface(MEMORY_INTER) %{
 5876     base($reg);
 5877     index(0xffffffff);
 5878     scale(0x0);
 5879     disp($off);
 5880   %}
 5881 %}
 5882 
 5883 operand indOffL16(iRegP reg, immLoffset16 off)
 5884 %{
 5885   constraint(ALLOC_IN_RC(ptr_reg));
 5886   match(AddP reg off);
 5887   op_cost(0);
 5888   format %{ "[$reg, $off]" %}
 5889   interface(MEMORY_INTER) %{
 5890     base($reg);
 5891     index(0xffffffff);
 5892     scale(0x0);
 5893     disp($off);
 5894   %}
 5895 %}
 5896 
 5897 operand indirectN(iRegN reg)
 5898 %{
 5899   predicate(CompressedOops::shift() == 0);
 5900   constraint(ALLOC_IN_RC(ptr_reg));
 5901   match(DecodeN reg);
 5902   op_cost(0);
 5903   format %{ "[$reg]\t# narrow" %}
 5904   interface(MEMORY_INTER) %{
 5905     base($reg);
 5906     index(0xffffffff);
 5907     scale(0x0);
 5908     disp(0x0);
 5909   %}
 5910 %}
 5911 
 5912 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5913 %{
 5914   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5915   constraint(ALLOC_IN_RC(ptr_reg));
 5916   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5917   op_cost(0);
 5918   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5919   interface(MEMORY_INTER) %{
 5920     base($reg);
 5921     index($ireg);
 5922     scale($scale);
 5923     disp(0x0);
 5924   %}
 5925 %}
 5926 
 5927 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5928 %{
 5929   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5930   constraint(ALLOC_IN_RC(ptr_reg));
 5931   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5932   op_cost(0);
 5933   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5934   interface(MEMORY_INTER) %{
 5935     base($reg);
 5936     index($lreg);
 5937     scale($scale);
 5938     disp(0x0);
 5939   %}
 5940 %}
 5941 
 5942 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5943 %{
 5944   predicate(CompressedOops::shift() == 0);
 5945   constraint(ALLOC_IN_RC(ptr_reg));
 5946   match(AddP (DecodeN reg) (ConvI2L ireg));
 5947   op_cost(0);
 5948   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5949   interface(MEMORY_INTER) %{
 5950     base($reg);
 5951     index($ireg);
 5952     scale(0x0);
 5953     disp(0x0);
 5954   %}
 5955 %}
 5956 
 5957 operand indIndexN(iRegN reg, iRegL lreg)
 5958 %{
 5959   predicate(CompressedOops::shift() == 0);
 5960   constraint(ALLOC_IN_RC(ptr_reg));
 5961   match(AddP (DecodeN reg) lreg);
 5962   op_cost(0);
 5963   format %{ "$reg, $lreg\t# narrow" %}
 5964   interface(MEMORY_INTER) %{
 5965     base($reg);
 5966     index($lreg);
 5967     scale(0x0);
 5968     disp(0x0);
 5969   %}
 5970 %}
 5971 
 5972 operand indOffIN(iRegN reg, immIOffset off)
 5973 %{
 5974   predicate(CompressedOops::shift() == 0);
 5975   constraint(ALLOC_IN_RC(ptr_reg));
 5976   match(AddP (DecodeN reg) off);
 5977   op_cost(0);
 5978   format %{ "[$reg, $off]\t# narrow" %}
 5979   interface(MEMORY_INTER) %{
 5980     base($reg);
 5981     index(0xffffffff);
 5982     scale(0x0);
 5983     disp($off);
 5984   %}
 5985 %}
 5986 
 5987 operand indOffLN(iRegN reg, immLoffset off)
 5988 %{
 5989   predicate(CompressedOops::shift() == 0);
 5990   constraint(ALLOC_IN_RC(ptr_reg));
 5991   match(AddP (DecodeN reg) off);
 5992   op_cost(0);
 5993   format %{ "[$reg, $off]\t# narrow" %}
 5994   interface(MEMORY_INTER) %{
 5995     base($reg);
 5996     index(0xffffffff);
 5997     scale(0x0);
 5998     disp($off);
 5999   %}
 6000 %}
 6001 
 6002 
 6003 
 6004 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6005 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6006 %{
 6007   constraint(ALLOC_IN_RC(ptr_reg));
 6008   match(AddP reg off);
 6009   op_cost(0);
 6010   format %{ "[$reg, $off]" %}
 6011   interface(MEMORY_INTER) %{
 6012     base($reg);
 6013     index(0xffffffff);
 6014     scale(0x0);
 6015     disp($off);
 6016   %}
 6017 %}
 6018 
 6019 //----------Special Memory Operands--------------------------------------------
 6020 // Stack Slot Operand - This operand is used for loading and storing temporary
 6021 //                      values on the stack where a match requires a value to
 6022 //                      flow through memory.
 6023 operand stackSlotP(sRegP reg)
 6024 %{
 6025   constraint(ALLOC_IN_RC(stack_slots));
 6026   op_cost(100);
 6027   // No match rule because this operand is only generated in matching
 6028   // match(RegP);
 6029   format %{ "[$reg]" %}
 6030   interface(MEMORY_INTER) %{
 6031     base(0x1e);  // RSP
 6032     index(0x0);  // No Index
 6033     scale(0x0);  // No Scale
 6034     disp($reg);  // Stack Offset
 6035   %}
 6036 %}
 6037 
 6038 operand stackSlotI(sRegI reg)
 6039 %{
 6040   constraint(ALLOC_IN_RC(stack_slots));
 6041   // No match rule because this operand is only generated in matching
 6042   // match(RegI);
 6043   format %{ "[$reg]" %}
 6044   interface(MEMORY_INTER) %{
 6045     base(0x1e);  // RSP
 6046     index(0x0);  // No Index
 6047     scale(0x0);  // No Scale
 6048     disp($reg);  // Stack Offset
 6049   %}
 6050 %}
 6051 
 6052 operand stackSlotF(sRegF reg)
 6053 %{
 6054   constraint(ALLOC_IN_RC(stack_slots));
 6055   // No match rule because this operand is only generated in matching
 6056   // match(RegF);
 6057   format %{ "[$reg]" %}
 6058   interface(MEMORY_INTER) %{
 6059     base(0x1e);  // RSP
 6060     index(0x0);  // No Index
 6061     scale(0x0);  // No Scale
 6062     disp($reg);  // Stack Offset
 6063   %}
 6064 %}
 6065 
 6066 operand stackSlotD(sRegD reg)
 6067 %{
 6068   constraint(ALLOC_IN_RC(stack_slots));
 6069   // No match rule because this operand is only generated in matching
 6070   // match(RegD);
 6071   format %{ "[$reg]" %}
 6072   interface(MEMORY_INTER) %{
 6073     base(0x1e);  // RSP
 6074     index(0x0);  // No Index
 6075     scale(0x0);  // No Scale
 6076     disp($reg);  // Stack Offset
 6077   %}
 6078 %}
 6079 
 6080 operand stackSlotL(sRegL reg)
 6081 %{
 6082   constraint(ALLOC_IN_RC(stack_slots));
 6083   // No match rule because this operand is only generated in matching
 6084   // match(RegL);
 6085   format %{ "[$reg]" %}
 6086   interface(MEMORY_INTER) %{
 6087     base(0x1e);  // RSP
 6088     index(0x0);  // No Index
 6089     scale(0x0);  // No Scale
 6090     disp($reg);  // Stack Offset
 6091   %}
 6092 %}
 6093 
 6094 // Operands for expressing Control Flow
 6095 // NOTE: Label is a predefined operand which should not be redefined in
 6096 //       the AD file. It is generically handled within the ADLC.
 6097 
 6098 //----------Conditional Branch Operands----------------------------------------
 6099 // Comparison Op  - This is the operation of the comparison, and is limited to
 6100 //                  the following set of codes:
 6101 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6102 //
 6103 // Other attributes of the comparison, such as unsignedness, are specified
 6104 // by the comparison instruction that sets a condition code flags register.
 6105 // That result is represented by a flags operand whose subtype is appropriate
 6106 // to the unsignedness (etc.) of the comparison.
 6107 //
 6108 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6109 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6110 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6111 
 6112 // used for signed integral comparisons and fp comparisons
 6113 
 6114 operand cmpOp()
 6115 %{
 6116   match(Bool);
 6117 
 6118   format %{ "" %}
 6119   interface(COND_INTER) %{
 6120     equal(0x0, "eq");
 6121     not_equal(0x1, "ne");
 6122     less(0xb, "lt");
 6123     greater_equal(0xa, "ge");
 6124     less_equal(0xd, "le");
 6125     greater(0xc, "gt");
 6126     overflow(0x6, "vs");
 6127     no_overflow(0x7, "vc");
 6128   %}
 6129 %}
 6130 
 6131 // used for unsigned integral comparisons
 6132 
 6133 operand cmpOpU()
 6134 %{
 6135   match(Bool);
 6136 
 6137   format %{ "" %}
 6138   interface(COND_INTER) %{
 6139     equal(0x0, "eq");
 6140     not_equal(0x1, "ne");
 6141     less(0x3, "lo");
 6142     greater_equal(0x2, "hs");
 6143     less_equal(0x9, "ls");
 6144     greater(0x8, "hi");
 6145     overflow(0x6, "vs");
 6146     no_overflow(0x7, "vc");
 6147   %}
 6148 %}
 6149 
 6150 // used for certain integral comparisons which can be
 6151 // converted to cbxx or tbxx instructions
 6152 
 6153 operand cmpOpEqNe()
 6154 %{
 6155   match(Bool);
 6156   op_cost(0);
 6157   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6158             || n->as_Bool()->_test._test == BoolTest::eq);
 6159 
 6160   format %{ "" %}
 6161   interface(COND_INTER) %{
 6162     equal(0x0, "eq");
 6163     not_equal(0x1, "ne");
 6164     less(0xb, "lt");
 6165     greater_equal(0xa, "ge");
 6166     less_equal(0xd, "le");
 6167     greater(0xc, "gt");
 6168     overflow(0x6, "vs");
 6169     no_overflow(0x7, "vc");
 6170   %}
 6171 %}
 6172 
 6173 // used for certain integral comparisons which can be
 6174 // converted to cbxx or tbxx instructions
 6175 
 6176 operand cmpOpLtGe()
 6177 %{
 6178   match(Bool);
 6179   op_cost(0);
 6180 
 6181   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6182             || n->as_Bool()->_test._test == BoolTest::ge);
 6183 
 6184   format %{ "" %}
 6185   interface(COND_INTER) %{
 6186     equal(0x0, "eq");
 6187     not_equal(0x1, "ne");
 6188     less(0xb, "lt");
 6189     greater_equal(0xa, "ge");
 6190     less_equal(0xd, "le");
 6191     greater(0xc, "gt");
 6192     overflow(0x6, "vs");
 6193     no_overflow(0x7, "vc");
 6194   %}
 6195 %}
 6196 
 6197 // used for certain unsigned integral comparisons which can be
 6198 // converted to cbxx or tbxx instructions
 6199 
 6200 operand cmpOpUEqNeLtGe()
 6201 %{
 6202   match(Bool);
 6203   op_cost(0);
 6204 
 6205   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6206             || n->as_Bool()->_test._test == BoolTest::ne
 6207             || n->as_Bool()->_test._test == BoolTest::lt
 6208             || n->as_Bool()->_test._test == BoolTest::ge);
 6209 
 6210   format %{ "" %}
 6211   interface(COND_INTER) %{
 6212     equal(0x0, "eq");
 6213     not_equal(0x1, "ne");
 6214     less(0xb, "lt");
 6215     greater_equal(0xa, "ge");
 6216     less_equal(0xd, "le");
 6217     greater(0xc, "gt");
 6218     overflow(0x6, "vs");
 6219     no_overflow(0x7, "vc");
 6220   %}
 6221 %}
 6222 
 6223 // Special operand allowing long args to int ops to be truncated for free
 6224 
 6225 operand iRegL2I(iRegL reg) %{
 6226 
 6227   op_cost(0);
 6228 
 6229   match(ConvL2I reg);
 6230 
 6231   format %{ "l2i($reg)" %}
 6232 
 6233   interface(REG_INTER)
 6234 %}
 6235 
 6236 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6237 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6238 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6239 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6240 
 6241 //----------OPERAND CLASSES----------------------------------------------------
 6242 // Operand Classes are groups of operands that are used as to simplify
 6243 // instruction definitions by not requiring the AD writer to specify
 6244 // separate instructions for every form of operand when the
 6245 // instruction accepts multiple operand types with the same basic
 6246 // encoding and format. The classic case of this is memory operands.
 6247 
 6248 // memory is used to define read/write location for load/store
 6249 // instruction defs. we can turn a memory op into an Address
 6250 
 6251 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6252                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6253 
 6254 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6255                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6256 
 6257 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6258                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6259 
 6260 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6261                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6262 
 6263 // All of the memory operands. For the pipeline description.
 6264 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6265                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6266                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6267 
 6268 
 6269 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6270 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6271 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6272 // can be elided because the 32-bit instruction will just employ the
 6273 // lower 32 bits anyway.
 6274 //
 6275 // n.b. this does not elide all L2I conversions. if the truncated
 6276 // value is consumed by more than one operation then the ConvL2I
 6277 // cannot be bundled into the consuming nodes so an l2i gets planted
 6278 // (actually a movw $dst $src) and the downstream instructions consume
 6279 // the result of the l2i as an iRegI input. That's a shame since the
 6280 // movw is actually redundant but its not too costly.
 6281 
 6282 opclass iRegIorL2I(iRegI, iRegL2I);
 6283 
 6284 //----------PIPELINE-----------------------------------------------------------
 6285 // Rules which define the behavior of the target architectures pipeline.
 6286 
 6287 // For specific pipelines, eg A53, define the stages of that pipeline
 6288 //pipe_desc(ISS, EX1, EX2, WR);
 6289 #define ISS S0
 6290 #define EX1 S1
 6291 #define EX2 S2
 6292 #define WR  S3
 6293 
 6294 // Integer ALU reg operation
 6295 pipeline %{
 6296 
 6297 attributes %{
 6298   // ARM instructions are of fixed length
 6299   fixed_size_instructions;        // Fixed size instructions TODO does
 6300   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6301   // ARM instructions come in 32-bit word units
 6302   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6303   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6304   instruction_fetch_units = 1;       // of 64 bytes
 6305 
 6306   // List of nop instructions
 6307   nops( MachNop );
 6308 %}
 6309 
 6310 // We don't use an actual pipeline model so don't care about resources
 6311 // or description. we do use pipeline classes to introduce fixed
 6312 // latencies
 6313 
 6314 //----------RESOURCES----------------------------------------------------------
 6315 // Resources are the functional units available to the machine
 6316 
 6317 resources( INS0, INS1, INS01 = INS0 | INS1,
 6318            ALU0, ALU1, ALU = ALU0 | ALU1,
 6319            MAC,
 6320            DIV,
 6321            BRANCH,
 6322            LDST,
 6323            NEON_FP);
 6324 
 6325 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6326 // Pipeline Description specifies the stages in the machine's pipeline
 6327 
 6328 // Define the pipeline as a generic 6 stage pipeline
 6329 pipe_desc(S0, S1, S2, S3, S4, S5);
 6330 
 6331 //----------PIPELINE CLASSES---------------------------------------------------
 6332 // Pipeline Classes describe the stages in which input and output are
 6333 // referenced by the hardware pipeline.
 6334 
 6335 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6336 %{
 6337   single_instruction;
 6338   src1   : S1(read);
 6339   src2   : S2(read);
 6340   dst    : S5(write);
 6341   INS01  : ISS;
 6342   NEON_FP : S5;
 6343 %}
 6344 
 6345 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6346 %{
 6347   single_instruction;
 6348   src1   : S1(read);
 6349   src2   : S2(read);
 6350   dst    : S5(write);
 6351   INS01  : ISS;
 6352   NEON_FP : S5;
 6353 %}
 6354 
 6355 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6356 %{
 6357   single_instruction;
 6358   src    : S1(read);
 6359   dst    : S5(write);
 6360   INS01  : ISS;
 6361   NEON_FP : S5;
 6362 %}
 6363 
 6364 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6365 %{
 6366   single_instruction;
 6367   src    : S1(read);
 6368   dst    : S5(write);
 6369   INS01  : ISS;
 6370   NEON_FP : S5;
 6371 %}
 6372 
 6373 pipe_class fp_d2f(vRegF dst, vRegD src)
 6374 %{
 6375   single_instruction;
 6376   src    : S1(read);
 6377   dst    : S5(write);
 6378   INS01  : ISS;
 6379   NEON_FP : S5;
 6380 %}
 6381 
 6382 pipe_class fp_f2d(vRegD dst, vRegF src)
 6383 %{
 6384   single_instruction;
 6385   src    : S1(read);
 6386   dst    : S5(write);
 6387   INS01  : ISS;
 6388   NEON_FP : S5;
 6389 %}
 6390 
 6391 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6392 %{
 6393   single_instruction;
 6394   src    : S1(read);
 6395   dst    : S5(write);
 6396   INS01  : ISS;
 6397   NEON_FP : S5;
 6398 %}
 6399 
 6400 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6401 %{
 6402   single_instruction;
 6403   src    : S1(read);
 6404   dst    : S5(write);
 6405   INS01  : ISS;
 6406   NEON_FP : S5;
 6407 %}
 6408 
 6409 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6410 %{
 6411   single_instruction;
 6412   src    : S1(read);
 6413   dst    : S5(write);
 6414   INS01  : ISS;
 6415   NEON_FP : S5;
 6416 %}
 6417 
 6418 pipe_class fp_l2f(vRegF dst, iRegL src)
 6419 %{
 6420   single_instruction;
 6421   src    : S1(read);
 6422   dst    : S5(write);
 6423   INS01  : ISS;
 6424   NEON_FP : S5;
 6425 %}
 6426 
 6427 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6428 %{
 6429   single_instruction;
 6430   src    : S1(read);
 6431   dst    : S5(write);
 6432   INS01  : ISS;
 6433   NEON_FP : S5;
 6434 %}
 6435 
 6436 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6437 %{
 6438   single_instruction;
 6439   src    : S1(read);
 6440   dst    : S5(write);
 6441   INS01  : ISS;
 6442   NEON_FP : S5;
 6443 %}
 6444 
 6445 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6446 %{
 6447   single_instruction;
 6448   src    : S1(read);
 6449   dst    : S5(write);
 6450   INS01  : ISS;
 6451   NEON_FP : S5;
 6452 %}
 6453 
 6454 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6455 %{
 6456   single_instruction;
 6457   src    : S1(read);
 6458   dst    : S5(write);
 6459   INS01  : ISS;
 6460   NEON_FP : S5;
 6461 %}
 6462 
 6463 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6464 %{
 6465   single_instruction;
 6466   src1   : S1(read);
 6467   src2   : S2(read);
 6468   dst    : S5(write);
 6469   INS0   : ISS;
 6470   NEON_FP : S5;
 6471 %}
 6472 
 6473 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6474 %{
 6475   single_instruction;
 6476   src1   : S1(read);
 6477   src2   : S2(read);
 6478   dst    : S5(write);
 6479   INS0   : ISS;
 6480   NEON_FP : S5;
 6481 %}
 6482 
 6483 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6484 %{
 6485   single_instruction;
 6486   cr     : S1(read);
 6487   src1   : S1(read);
 6488   src2   : S1(read);
 6489   dst    : S3(write);
 6490   INS01  : ISS;
 6491   NEON_FP : S3;
 6492 %}
 6493 
 6494 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6495 %{
 6496   single_instruction;
 6497   cr     : S1(read);
 6498   src1   : S1(read);
 6499   src2   : S1(read);
 6500   dst    : S3(write);
 6501   INS01  : ISS;
 6502   NEON_FP : S3;
 6503 %}
 6504 
 6505 pipe_class fp_imm_s(vRegF dst)
 6506 %{
 6507   single_instruction;
 6508   dst    : S3(write);
 6509   INS01  : ISS;
 6510   NEON_FP : S3;
 6511 %}
 6512 
 6513 pipe_class fp_imm_d(vRegD dst)
 6514 %{
 6515   single_instruction;
 6516   dst    : S3(write);
 6517   INS01  : ISS;
 6518   NEON_FP : S3;
 6519 %}
 6520 
 6521 pipe_class fp_load_constant_s(vRegF dst)
 6522 %{
 6523   single_instruction;
 6524   dst    : S4(write);
 6525   INS01  : ISS;
 6526   NEON_FP : S4;
 6527 %}
 6528 
 6529 pipe_class fp_load_constant_d(vRegD dst)
 6530 %{
 6531   single_instruction;
 6532   dst    : S4(write);
 6533   INS01  : ISS;
 6534   NEON_FP : S4;
 6535 %}
 6536 
 6537 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6538 %{
 6539   single_instruction;
 6540   dst    : S5(write);
 6541   src1   : S1(read);
 6542   src2   : S1(read);
 6543   INS01  : ISS;
 6544   NEON_FP : S5;
 6545 %}
 6546 
 6547 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6548 %{
 6549   single_instruction;
 6550   dst    : S5(write);
 6551   src1   : S1(read);
 6552   src2   : S1(read);
 6553   INS0   : ISS;
 6554   NEON_FP : S5;
 6555 %}
 6556 
 6557 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6558 %{
 6559   single_instruction;
 6560   dst    : S5(write);
 6561   src1   : S1(read);
 6562   src2   : S1(read);
 6563   dst    : S1(read);
 6564   INS01  : ISS;
 6565   NEON_FP : S5;
 6566 %}
 6567 
 6568 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6569 %{
 6570   single_instruction;
 6571   dst    : S5(write);
 6572   src1   : S1(read);
 6573   src2   : S1(read);
 6574   dst    : S1(read);
 6575   INS0   : ISS;
 6576   NEON_FP : S5;
 6577 %}
 6578 
 6579 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6580 %{
 6581   single_instruction;
 6582   dst    : S4(write);
 6583   src1   : S2(read);
 6584   src2   : S2(read);
 6585   INS01  : ISS;
 6586   NEON_FP : S4;
 6587 %}
 6588 
 6589 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6590 %{
 6591   single_instruction;
 6592   dst    : S4(write);
 6593   src1   : S2(read);
 6594   src2   : S2(read);
 6595   INS0   : ISS;
 6596   NEON_FP : S4;
 6597 %}
 6598 
 6599 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6600 %{
 6601   single_instruction;
 6602   dst    : S3(write);
 6603   src1   : S2(read);
 6604   src2   : S2(read);
 6605   INS01  : ISS;
 6606   NEON_FP : S3;
 6607 %}
 6608 
 6609 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6610 %{
 6611   single_instruction;
 6612   dst    : S3(write);
 6613   src1   : S2(read);
 6614   src2   : S2(read);
 6615   INS0   : ISS;
 6616   NEON_FP : S3;
 6617 %}
 6618 
 6619 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6620 %{
 6621   single_instruction;
 6622   dst    : S3(write);
 6623   src    : S1(read);
 6624   shift  : S1(read);
 6625   INS01  : ISS;
 6626   NEON_FP : S3;
 6627 %}
 6628 
 6629 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6630 %{
 6631   single_instruction;
 6632   dst    : S3(write);
 6633   src    : S1(read);
 6634   shift  : S1(read);
 6635   INS0   : ISS;
 6636   NEON_FP : S3;
 6637 %}
 6638 
 6639 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6640 %{
 6641   single_instruction;
 6642   dst    : S3(write);
 6643   src    : S1(read);
 6644   INS01  : ISS;
 6645   NEON_FP : S3;
 6646 %}
 6647 
 6648 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6649 %{
 6650   single_instruction;
 6651   dst    : S3(write);
 6652   src    : S1(read);
 6653   INS0   : ISS;
 6654   NEON_FP : S3;
 6655 %}
 6656 
 6657 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6658 %{
 6659   single_instruction;
 6660   dst    : S5(write);
 6661   src1   : S1(read);
 6662   src2   : S1(read);
 6663   INS01  : ISS;
 6664   NEON_FP : S5;
 6665 %}
 6666 
 6667 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6668 %{
 6669   single_instruction;
 6670   dst    : S5(write);
 6671   src1   : S1(read);
 6672   src2   : S1(read);
 6673   INS0   : ISS;
 6674   NEON_FP : S5;
 6675 %}
 6676 
 6677 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6678 %{
 6679   single_instruction;
 6680   dst    : S5(write);
 6681   src1   : S1(read);
 6682   src2   : S1(read);
 6683   INS0   : ISS;
 6684   NEON_FP : S5;
 6685 %}
 6686 
 6687 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6688 %{
 6689   single_instruction;
 6690   dst    : S5(write);
 6691   src1   : S1(read);
 6692   src2   : S1(read);
 6693   INS0   : ISS;
 6694   NEON_FP : S5;
 6695 %}
 6696 
 6697 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6698 %{
 6699   single_instruction;
 6700   dst    : S5(write);
 6701   src    : S1(read);
 6702   INS0   : ISS;
 6703   NEON_FP : S5;
 6704 %}
 6705 
 6706 pipe_class vunop_fp64(vecD dst, vecD src)
 6707 %{
 6708   single_instruction;
 6709   dst    : S5(write);
 6710   src    : S1(read);
 6711   INS01  : ISS;
 6712   NEON_FP : S5;
 6713 %}
 6714 
 6715 pipe_class vunop_fp128(vecX dst, vecX src)
 6716 %{
 6717   single_instruction;
 6718   dst    : S5(write);
 6719   src    : S1(read);
 6720   INS0   : ISS;
 6721   NEON_FP : S5;
 6722 %}
 6723 
 6724 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6725 %{
 6726   single_instruction;
 6727   dst    : S3(write);
 6728   src    : S1(read);
 6729   INS01  : ISS;
 6730   NEON_FP : S3;
 6731 %}
 6732 
 6733 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6734 %{
 6735   single_instruction;
 6736   dst    : S3(write);
 6737   src    : S1(read);
 6738   INS01  : ISS;
 6739   NEON_FP : S3;
 6740 %}
 6741 
 6742 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6743 %{
 6744   single_instruction;
 6745   dst    : S3(write);
 6746   src    : S1(read);
 6747   INS01  : ISS;
 6748   NEON_FP : S3;
 6749 %}
 6750 
 6751 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6752 %{
 6753   single_instruction;
 6754   dst    : S3(write);
 6755   src    : S1(read);
 6756   INS01  : ISS;
 6757   NEON_FP : S3;
 6758 %}
 6759 
 6760 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6761 %{
 6762   single_instruction;
 6763   dst    : S3(write);
 6764   src    : S1(read);
 6765   INS01  : ISS;
 6766   NEON_FP : S3;
 6767 %}
 6768 
 6769 pipe_class vmovi_reg_imm64(vecD dst)
 6770 %{
 6771   single_instruction;
 6772   dst    : S3(write);
 6773   INS01  : ISS;
 6774   NEON_FP : S3;
 6775 %}
 6776 
 6777 pipe_class vmovi_reg_imm128(vecX dst)
 6778 %{
 6779   single_instruction;
 6780   dst    : S3(write);
 6781   INS0   : ISS;
 6782   NEON_FP : S3;
 6783 %}
 6784 
 6785 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6786 %{
 6787   single_instruction;
 6788   dst    : S5(write);
 6789   mem    : ISS(read);
 6790   INS01  : ISS;
 6791   NEON_FP : S3;
 6792 %}
 6793 
 6794 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6795 %{
 6796   single_instruction;
 6797   dst    : S5(write);
 6798   mem    : ISS(read);
 6799   INS01  : ISS;
 6800   NEON_FP : S3;
 6801 %}
 6802 
 6803 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6804 %{
 6805   single_instruction;
 6806   mem    : ISS(read);
 6807   src    : S2(read);
 6808   INS01  : ISS;
 6809   NEON_FP : S3;
 6810 %}
 6811 
 6812 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6813 %{
 6814   single_instruction;
 6815   mem    : ISS(read);
 6816   src    : S2(read);
 6817   INS01  : ISS;
 6818   NEON_FP : S3;
 6819 %}
 6820 
 6821 //------- Integer ALU operations --------------------------
 6822 
 6823 // Integer ALU reg-reg operation
 6824 // Operands needed in EX1, result generated in EX2
 6825 // Eg.  ADD     x0, x1, x2
 6826 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6827 %{
 6828   single_instruction;
 6829   dst    : EX2(write);
 6830   src1   : EX1(read);
 6831   src2   : EX1(read);
 6832   INS01  : ISS; // Dual issue as instruction 0 or 1
 6833   ALU    : EX2;
 6834 %}
 6835 
 6836 // Integer ALU reg-reg operation with constant shift
 6837 // Shifted register must be available in LATE_ISS instead of EX1
 6838 // Eg.  ADD     x0, x1, x2, LSL #2
 6839 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6840 %{
 6841   single_instruction;
 6842   dst    : EX2(write);
 6843   src1   : EX1(read);
 6844   src2   : ISS(read);
 6845   INS01  : ISS;
 6846   ALU    : EX2;
 6847 %}
 6848 
 6849 // Integer ALU reg operation with constant shift
 6850 // Eg.  LSL     x0, x1, #shift
 6851 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6852 %{
 6853   single_instruction;
 6854   dst    : EX2(write);
 6855   src1   : ISS(read);
 6856   INS01  : ISS;
 6857   ALU    : EX2;
 6858 %}
 6859 
 6860 // Integer ALU reg-reg operation with variable shift
 6861 // Both operands must be available in LATE_ISS instead of EX1
 6862 // Result is available in EX1 instead of EX2
 6863 // Eg.  LSLV    x0, x1, x2
 6864 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6865 %{
 6866   single_instruction;
 6867   dst    : EX1(write);
 6868   src1   : ISS(read);
 6869   src2   : ISS(read);
 6870   INS01  : ISS;
 6871   ALU    : EX1;
 6872 %}
 6873 
 6874 // Integer ALU reg-reg operation with extract
 6875 // As for _vshift above, but result generated in EX2
 6876 // Eg.  EXTR    x0, x1, x2, #N
 6877 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6878 %{
 6879   single_instruction;
 6880   dst    : EX2(write);
 6881   src1   : ISS(read);
 6882   src2   : ISS(read);
 6883   INS1   : ISS; // Can only dual issue as Instruction 1
 6884   ALU    : EX1;
 6885 %}
 6886 
 6887 // Integer ALU reg operation
 6888 // Eg.  NEG     x0, x1
 6889 pipe_class ialu_reg(iRegI dst, iRegI src)
 6890 %{
 6891   single_instruction;
 6892   dst    : EX2(write);
 6893   src    : EX1(read);
 6894   INS01  : ISS;
 6895   ALU    : EX2;
 6896 %}
 6897 
 6898 // Integer ALU reg mmediate operation
 6899 // Eg.  ADD     x0, x1, #N
 6900 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6901 %{
 6902   single_instruction;
 6903   dst    : EX2(write);
 6904   src1   : EX1(read);
 6905   INS01  : ISS;
 6906   ALU    : EX2;
 6907 %}
 6908 
 6909 // Integer ALU immediate operation (no source operands)
 6910 // Eg.  MOV     x0, #N
 6911 pipe_class ialu_imm(iRegI dst)
 6912 %{
 6913   single_instruction;
 6914   dst    : EX1(write);
 6915   INS01  : ISS;
 6916   ALU    : EX1;
 6917 %}
 6918 
 6919 //------- Compare operation -------------------------------
 6920 
 6921 // Compare reg-reg
 6922 // Eg.  CMP     x0, x1
 6923 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6924 %{
 6925   single_instruction;
 6926 //  fixed_latency(16);
 6927   cr     : EX2(write);
 6928   op1    : EX1(read);
 6929   op2    : EX1(read);
 6930   INS01  : ISS;
 6931   ALU    : EX2;
 6932 %}
 6933 
 6934 // Compare reg-reg
 6935 // Eg.  CMP     x0, #N
 6936 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6937 %{
 6938   single_instruction;
 6939 //  fixed_latency(16);
 6940   cr     : EX2(write);
 6941   op1    : EX1(read);
 6942   INS01  : ISS;
 6943   ALU    : EX2;
 6944 %}
 6945 
 6946 //------- Conditional instructions ------------------------
 6947 
 6948 // Conditional no operands
 6949 // Eg.  CSINC   x0, zr, zr, <cond>
 6950 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6951 %{
 6952   single_instruction;
 6953   cr     : EX1(read);
 6954   dst    : EX2(write);
 6955   INS01  : ISS;
 6956   ALU    : EX2;
 6957 %}
 6958 
 6959 // Conditional 2 operand
 6960 // EG.  CSEL    X0, X1, X2, <cond>
 6961 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6962 %{
 6963   single_instruction;
 6964   cr     : EX1(read);
 6965   src1   : EX1(read);
 6966   src2   : EX1(read);
 6967   dst    : EX2(write);
 6968   INS01  : ISS;
 6969   ALU    : EX2;
 6970 %}
 6971 
 6972 // Conditional 2 operand
 6973 // EG.  CSEL    X0, X1, X2, <cond>
 6974 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6975 %{
 6976   single_instruction;
 6977   cr     : EX1(read);
 6978   src    : EX1(read);
 6979   dst    : EX2(write);
 6980   INS01  : ISS;
 6981   ALU    : EX2;
 6982 %}
 6983 
 6984 //------- Multiply pipeline operations --------------------
 6985 
 6986 // Multiply reg-reg
 6987 // Eg.  MUL     w0, w1, w2
 6988 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6989 %{
 6990   single_instruction;
 6991   dst    : WR(write);
 6992   src1   : ISS(read);
 6993   src2   : ISS(read);
 6994   INS01  : ISS;
 6995   MAC    : WR;
 6996 %}
 6997 
 6998 // Multiply accumulate
 6999 // Eg.  MADD    w0, w1, w2, w3
 7000 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7001 %{
 7002   single_instruction;
 7003   dst    : WR(write);
 7004   src1   : ISS(read);
 7005   src2   : ISS(read);
 7006   src3   : ISS(read);
 7007   INS01  : ISS;
 7008   MAC    : WR;
 7009 %}
 7010 
 7011 // Eg.  MUL     w0, w1, w2
 7012 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7013 %{
 7014   single_instruction;
 7015   fixed_latency(3); // Maximum latency for 64 bit mul
 7016   dst    : WR(write);
 7017   src1   : ISS(read);
 7018   src2   : ISS(read);
 7019   INS01  : ISS;
 7020   MAC    : WR;
 7021 %}
 7022 
 7023 // Multiply accumulate
 7024 // Eg.  MADD    w0, w1, w2, w3
 7025 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7026 %{
 7027   single_instruction;
 7028   fixed_latency(3); // Maximum latency for 64 bit mul
 7029   dst    : WR(write);
 7030   src1   : ISS(read);
 7031   src2   : ISS(read);
 7032   src3   : ISS(read);
 7033   INS01  : ISS;
 7034   MAC    : WR;
 7035 %}
 7036 
 7037 //------- Divide pipeline operations --------------------
 7038 
 7039 // Eg.  SDIV    w0, w1, w2
 7040 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7041 %{
 7042   single_instruction;
 7043   fixed_latency(8); // Maximum latency for 32 bit divide
 7044   dst    : WR(write);
 7045   src1   : ISS(read);
 7046   src2   : ISS(read);
 7047   INS0   : ISS; // Can only dual issue as instruction 0
 7048   DIV    : WR;
 7049 %}
 7050 
 7051 // Eg.  SDIV    x0, x1, x2
 7052 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7053 %{
 7054   single_instruction;
 7055   fixed_latency(16); // Maximum latency for 64 bit divide
 7056   dst    : WR(write);
 7057   src1   : ISS(read);
 7058   src2   : ISS(read);
 7059   INS0   : ISS; // Can only dual issue as instruction 0
 7060   DIV    : WR;
 7061 %}
 7062 
 7063 //------- Load pipeline operations ------------------------
 7064 
 7065 // Load - prefetch
 7066 // Eg.  PFRM    <mem>
 7067 pipe_class iload_prefetch(memory mem)
 7068 %{
 7069   single_instruction;
 7070   mem    : ISS(read);
 7071   INS01  : ISS;
 7072   LDST   : WR;
 7073 %}
 7074 
 7075 // Load - reg, mem
 7076 // Eg.  LDR     x0, <mem>
 7077 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7078 %{
 7079   single_instruction;
 7080   dst    : WR(write);
 7081   mem    : ISS(read);
 7082   INS01  : ISS;
 7083   LDST   : WR;
 7084 %}
 7085 
 7086 // Load - reg, reg
 7087 // Eg.  LDR     x0, [sp, x1]
 7088 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7089 %{
 7090   single_instruction;
 7091   dst    : WR(write);
 7092   src    : ISS(read);
 7093   INS01  : ISS;
 7094   LDST   : WR;
 7095 %}
 7096 
 7097 //------- Store pipeline operations -----------------------
 7098 
 7099 // Store - zr, mem
 7100 // Eg.  STR     zr, <mem>
 7101 pipe_class istore_mem(memory mem)
 7102 %{
 7103   single_instruction;
 7104   mem    : ISS(read);
 7105   INS01  : ISS;
 7106   LDST   : WR;
 7107 %}
 7108 
 7109 // Store - reg, mem
 7110 // Eg.  STR     x0, <mem>
 7111 pipe_class istore_reg_mem(iRegI src, memory mem)
 7112 %{
 7113   single_instruction;
 7114   mem    : ISS(read);
 7115   src    : EX2(read);
 7116   INS01  : ISS;
 7117   LDST   : WR;
 7118 %}
 7119 
 7120 // Store - reg, reg
 7121 // Eg. STR      x0, [sp, x1]
 7122 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7123 %{
 7124   single_instruction;
 7125   dst    : ISS(read);
 7126   src    : EX2(read);
 7127   INS01  : ISS;
 7128   LDST   : WR;
 7129 %}
 7130 
 7131 //------- Store pipeline operations -----------------------
 7132 
 7133 // Branch
 7134 pipe_class pipe_branch()
 7135 %{
 7136   single_instruction;
 7137   INS01  : ISS;
 7138   BRANCH : EX1;
 7139 %}
 7140 
 7141 // Conditional branch
 7142 pipe_class pipe_branch_cond(rFlagsReg cr)
 7143 %{
 7144   single_instruction;
 7145   cr     : EX1(read);
 7146   INS01  : ISS;
 7147   BRANCH : EX1;
 7148 %}
 7149 
 7150 // Compare & Branch
 7151 // EG.  CBZ/CBNZ
 7152 pipe_class pipe_cmp_branch(iRegI op1)
 7153 %{
 7154   single_instruction;
 7155   op1    : EX1(read);
 7156   INS01  : ISS;
 7157   BRANCH : EX1;
 7158 %}
 7159 
 7160 //------- Synchronisation operations ----------------------
 7161 
 7162 // Any operation requiring serialization.
 7163 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7164 pipe_class pipe_serial()
 7165 %{
 7166   single_instruction;
 7167   force_serialization;
 7168   fixed_latency(16);
 7169   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7170   LDST   : WR;
 7171 %}
 7172 
 7173 // Generic big/slow expanded idiom - also serialized
 7174 pipe_class pipe_slow()
 7175 %{
 7176   instruction_count(10);
 7177   multiple_bundles;
 7178   force_serialization;
 7179   fixed_latency(16);
 7180   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7181   LDST   : WR;
 7182 %}
 7183 
 7184 // Empty pipeline class
 7185 pipe_class pipe_class_empty()
 7186 %{
 7187   single_instruction;
 7188   fixed_latency(0);
 7189 %}
 7190 
 7191 // Default pipeline class.
 7192 pipe_class pipe_class_default()
 7193 %{
 7194   single_instruction;
 7195   fixed_latency(2);
 7196 %}
 7197 
 7198 // Pipeline class for compares.
 7199 pipe_class pipe_class_compare()
 7200 %{
 7201   single_instruction;
 7202   fixed_latency(16);
 7203 %}
 7204 
 7205 // Pipeline class for memory operations.
 7206 pipe_class pipe_class_memory()
 7207 %{
 7208   single_instruction;
 7209   fixed_latency(16);
 7210 %}
 7211 
 7212 // Pipeline class for call.
 7213 pipe_class pipe_class_call()
 7214 %{
 7215   single_instruction;
 7216   fixed_latency(100);
 7217 %}
 7218 
 7219 // Define the class for the Nop node.
 7220 define %{
 7221    MachNop = pipe_class_empty;
 7222 %}
 7223 
 7224 %}
 7225 //----------INSTRUCTIONS-------------------------------------------------------
 7226 //
 7227 // match      -- States which machine-independent subtree may be replaced
 7228 //               by this instruction.
 7229 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7230 //               selection to identify a minimum cost tree of machine
 7231 //               instructions that matches a tree of machine-independent
 7232 //               instructions.
 7233 // format     -- A string providing the disassembly for this instruction.
 7234 //               The value of an instruction's operand may be inserted
 7235 //               by referring to it with a '$' prefix.
 7236 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7237 //               to within an encode class as $primary, $secondary, and $tertiary
 7238 //               rrspectively.  The primary opcode is commonly used to
 7239 //               indicate the type of machine instruction, while secondary
 7240 //               and tertiary are often used for prefix options or addressing
 7241 //               modes.
 7242 // ins_encode -- A list of encode classes with parameters. The encode class
 7243 //               name must have been defined in an 'enc_class' specification
 7244 //               in the encode section of the architecture description.
 7245 
 7246 // ============================================================================
 7247 // Memory (Load/Store) Instructions
 7248 
 7249 // Load Instructions
 7250 
 7251 // Load Byte (8 bit signed)
 7252 instruct loadB(iRegINoSp dst, memory1 mem)
 7253 %{
 7254   match(Set dst (LoadB mem));
 7255   predicate(!needs_acquiring_load(n));
 7256 
 7257   ins_cost(4 * INSN_COST);
 7258   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7259 
 7260   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7261 
 7262   ins_pipe(iload_reg_mem);
 7263 %}
 7264 
 7265 // Load Byte (8 bit signed) into long
 7266 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7267 %{
 7268   match(Set dst (ConvI2L (LoadB mem)));
 7269   predicate(!needs_acquiring_load(n->in(1)));
 7270 
 7271   ins_cost(4 * INSN_COST);
 7272   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7273 
 7274   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7275 
 7276   ins_pipe(iload_reg_mem);
 7277 %}
 7278 
 7279 // Load Byte (8 bit unsigned)
 7280 instruct loadUB(iRegINoSp dst, memory1 mem)
 7281 %{
 7282   match(Set dst (LoadUB mem));
 7283   predicate(!needs_acquiring_load(n));
 7284 
 7285   ins_cost(4 * INSN_COST);
 7286   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7287 
 7288   ins_encode(aarch64_enc_ldrb(dst, mem));
 7289 
 7290   ins_pipe(iload_reg_mem);
 7291 %}
 7292 
 7293 // Load Byte (8 bit unsigned) into long
 7294 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7295 %{
 7296   match(Set dst (ConvI2L (LoadUB mem)));
 7297   predicate(!needs_acquiring_load(n->in(1)));
 7298 
 7299   ins_cost(4 * INSN_COST);
 7300   format %{ "ldrb  $dst, $mem\t# byte" %}
 7301 
 7302   ins_encode(aarch64_enc_ldrb(dst, mem));
 7303 
 7304   ins_pipe(iload_reg_mem);
 7305 %}
 7306 
 7307 // Load Short (16 bit signed)
 7308 instruct loadS(iRegINoSp dst, memory2 mem)
 7309 %{
 7310   match(Set dst (LoadS mem));
 7311   predicate(!needs_acquiring_load(n));
 7312 
 7313   ins_cost(4 * INSN_COST);
 7314   format %{ "ldrshw  $dst, $mem\t# short" %}
 7315 
 7316   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7317 
 7318   ins_pipe(iload_reg_mem);
 7319 %}
 7320 
 7321 // Load Short (16 bit signed) into long
 7322 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7323 %{
 7324   match(Set dst (ConvI2L (LoadS mem)));
 7325   predicate(!needs_acquiring_load(n->in(1)));
 7326 
 7327   ins_cost(4 * INSN_COST);
 7328   format %{ "ldrsh  $dst, $mem\t# short" %}
 7329 
 7330   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7331 
 7332   ins_pipe(iload_reg_mem);
 7333 %}
 7334 
 7335 // Load Char (16 bit unsigned)
 7336 instruct loadUS(iRegINoSp dst, memory2 mem)
 7337 %{
 7338   match(Set dst (LoadUS mem));
 7339   predicate(!needs_acquiring_load(n));
 7340 
 7341   ins_cost(4 * INSN_COST);
 7342   format %{ "ldrh  $dst, $mem\t# short" %}
 7343 
 7344   ins_encode(aarch64_enc_ldrh(dst, mem));
 7345 
 7346   ins_pipe(iload_reg_mem);
 7347 %}
 7348 
 7349 // Load Short/Char (16 bit unsigned) into long
 7350 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7351 %{
 7352   match(Set dst (ConvI2L (LoadUS mem)));
 7353   predicate(!needs_acquiring_load(n->in(1)));
 7354 
 7355   ins_cost(4 * INSN_COST);
 7356   format %{ "ldrh  $dst, $mem\t# short" %}
 7357 
 7358   ins_encode(aarch64_enc_ldrh(dst, mem));
 7359 
 7360   ins_pipe(iload_reg_mem);
 7361 %}
 7362 
 7363 // Load Integer (32 bit signed)
 7364 instruct loadI(iRegINoSp dst, memory4 mem)
 7365 %{
 7366   match(Set dst (LoadI mem));
 7367   predicate(!needs_acquiring_load(n));
 7368 
 7369   ins_cost(4 * INSN_COST);
 7370   format %{ "ldrw  $dst, $mem\t# int" %}
 7371 
 7372   ins_encode(aarch64_enc_ldrw(dst, mem));
 7373 
 7374   ins_pipe(iload_reg_mem);
 7375 %}
 7376 
 7377 // Load Integer (32 bit signed) into long
 7378 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7379 %{
 7380   match(Set dst (ConvI2L (LoadI mem)));
 7381   predicate(!needs_acquiring_load(n->in(1)));
 7382 
 7383   ins_cost(4 * INSN_COST);
 7384   format %{ "ldrsw  $dst, $mem\t# int" %}
 7385 
 7386   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7387 
 7388   ins_pipe(iload_reg_mem);
 7389 %}
 7390 
 7391 // Load Integer (32 bit unsigned) into long
 7392 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7393 %{
 7394   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7395   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7396 
 7397   ins_cost(4 * INSN_COST);
 7398   format %{ "ldrw  $dst, $mem\t# int" %}
 7399 
 7400   ins_encode(aarch64_enc_ldrw(dst, mem));
 7401 
 7402   ins_pipe(iload_reg_mem);
 7403 %}
 7404 
 7405 // Load Long (64 bit signed)
 7406 instruct loadL(iRegLNoSp dst, memory8 mem)
 7407 %{
 7408   match(Set dst (LoadL mem));
 7409   predicate(!needs_acquiring_load(n));
 7410 
 7411   ins_cost(4 * INSN_COST);
 7412   format %{ "ldr  $dst, $mem\t# int" %}
 7413 
 7414   ins_encode(aarch64_enc_ldr(dst, mem));
 7415 
 7416   ins_pipe(iload_reg_mem);
 7417 %}
 7418 
 7419 // Load Range
 7420 instruct loadRange(iRegINoSp dst, memory4 mem)
 7421 %{
 7422   match(Set dst (LoadRange mem));
 7423 
 7424   ins_cost(4 * INSN_COST);
 7425   format %{ "ldrw  $dst, $mem\t# range" %}
 7426 
 7427   ins_encode(aarch64_enc_ldrw(dst, mem));
 7428 
 7429   ins_pipe(iload_reg_mem);
 7430 %}
 7431 
 7432 // Load Pointer
 7433 instruct loadP(iRegPNoSp dst, memory8 mem)
 7434 %{
 7435   match(Set dst (LoadP mem));
 7436   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7437 
 7438   ins_cost(4 * INSN_COST);
 7439   format %{ "ldr  $dst, $mem\t# ptr" %}
 7440 
 7441   ins_encode(aarch64_enc_ldr(dst, mem));
 7442 
 7443   ins_pipe(iload_reg_mem);
 7444 %}
 7445 
 7446 // Load Compressed Pointer
 7447 instruct loadN(iRegNNoSp dst, memory4 mem)
 7448 %{
 7449   match(Set dst (LoadN mem));
 7450   predicate(!needs_acquiring_load(n));
 7451 
 7452   ins_cost(4 * INSN_COST);
 7453   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7454 
 7455   ins_encode(aarch64_enc_ldrw(dst, mem));
 7456 
 7457   ins_pipe(iload_reg_mem);
 7458 %}
 7459 
 7460 // Load Klass Pointer
 7461 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7462 %{
 7463   match(Set dst (LoadKlass mem));
 7464   predicate(!needs_acquiring_load(n));
 7465 
 7466   ins_cost(4 * INSN_COST);
 7467   format %{ "ldr  $dst, $mem\t# class" %}
 7468 
 7469   ins_encode(aarch64_enc_ldr(dst, mem));
 7470 
 7471   ins_pipe(iload_reg_mem);
 7472 %}
 7473 
 7474 // Load Narrow Klass Pointer
 7475 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7476 %{
 7477   match(Set dst (LoadNKlass mem));
 7478   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 7479 
 7480   ins_cost(4 * INSN_COST);
 7481   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7482 
 7483   ins_encode(aarch64_enc_ldrw(dst, mem));
 7484 
 7485   ins_pipe(iload_reg_mem);
 7486 %}
 7487 
 7488 instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7489 %{
 7490   match(Set dst (LoadNKlass mem));
 7491   effect(KILL cr);
 7492   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 7493 
 7494   ins_cost(4 * INSN_COST);
 7495   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7496   ins_encode %{
 7497     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7498     assert($mem$$index$$Register == noreg, "expect no index");
 7499     Register dst = $dst$$Register;
 7500     Register obj = $mem$$base$$Register;
 7501     C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
 7502     Compile::current()->output()->add_stub(stub);
 7503     __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
 7504     // NOTE: We can't use tbnz here, because the target is sometimes too far away
 7505     // and cannot be encoded.
 7506     __ tst(dst, markWord::monitor_value);
 7507     __ br(Assembler::NE, stub->entry());
 7508     __ bind(stub->continuation());
 7509     __ lsr(dst, dst, markWord::klass_shift);
 7510   %}
 7511   ins_pipe(pipe_slow);
 7512 %}
 7513 
 7514 // Load Float
 7515 instruct loadF(vRegF dst, memory4 mem)
 7516 %{
 7517   match(Set dst (LoadF mem));
 7518   predicate(!needs_acquiring_load(n));
 7519 
 7520   ins_cost(4 * INSN_COST);
 7521   format %{ "ldrs  $dst, $mem\t# float" %}
 7522 
 7523   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7524 
 7525   ins_pipe(pipe_class_memory);
 7526 %}
 7527 
 7528 // Load Double
 7529 instruct loadD(vRegD dst, memory8 mem)
 7530 %{
 7531   match(Set dst (LoadD mem));
 7532   predicate(!needs_acquiring_load(n));
 7533 
 7534   ins_cost(4 * INSN_COST);
 7535   format %{ "ldrd  $dst, $mem\t# double" %}
 7536 
 7537   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7538 
 7539   ins_pipe(pipe_class_memory);
 7540 %}
 7541 
 7542 
 7543 // Load Int Constant
 7544 instruct loadConI(iRegINoSp dst, immI src)
 7545 %{
 7546   match(Set dst src);
 7547 
 7548   ins_cost(INSN_COST);
 7549   format %{ "mov $dst, $src\t# int" %}
 7550 
 7551   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7552 
 7553   ins_pipe(ialu_imm);
 7554 %}
 7555 
 7556 // Load Long Constant
 7557 instruct loadConL(iRegLNoSp dst, immL src)
 7558 %{
 7559   match(Set dst src);
 7560 
 7561   ins_cost(INSN_COST);
 7562   format %{ "mov $dst, $src\t# long" %}
 7563 
 7564   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7565 
 7566   ins_pipe(ialu_imm);
 7567 %}
 7568 
 7569 // Load Pointer Constant
 7570 
 7571 instruct loadConP(iRegPNoSp dst, immP con)
 7572 %{
 7573   match(Set dst con);
 7574 
 7575   ins_cost(INSN_COST * 4);
 7576   format %{
 7577     "mov  $dst, $con\t# ptr\n\t"
 7578   %}
 7579 
 7580   ins_encode(aarch64_enc_mov_p(dst, con));
 7581 
 7582   ins_pipe(ialu_imm);
 7583 %}
 7584 
 7585 // Load Null Pointer Constant
 7586 
 7587 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7588 %{
 7589   match(Set dst con);
 7590 
 7591   ins_cost(INSN_COST);
 7592   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7593 
 7594   ins_encode(aarch64_enc_mov_p0(dst, con));
 7595 
 7596   ins_pipe(ialu_imm);
 7597 %}
 7598 
 7599 // Load Pointer Constant One
 7600 
 7601 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7602 %{
 7603   match(Set dst con);
 7604 
 7605   ins_cost(INSN_COST);
 7606   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7607 
 7608   ins_encode(aarch64_enc_mov_p1(dst, con));
 7609 
 7610   ins_pipe(ialu_imm);
 7611 %}
 7612 
 7613 // Load Byte Map Base Constant
 7614 
 7615 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7616 %{
 7617   match(Set dst con);
 7618 
 7619   ins_cost(INSN_COST);
 7620   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7621 
 7622   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7623 
 7624   ins_pipe(ialu_imm);
 7625 %}
 7626 
 7627 // Load Narrow Pointer Constant
 7628 
 7629 instruct loadConN(iRegNNoSp dst, immN con)
 7630 %{
 7631   match(Set dst con);
 7632 
 7633   ins_cost(INSN_COST * 4);
 7634   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7635 
 7636   ins_encode(aarch64_enc_mov_n(dst, con));
 7637 
 7638   ins_pipe(ialu_imm);
 7639 %}
 7640 
 7641 // Load Narrow Null Pointer Constant
 7642 
 7643 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7644 %{
 7645   match(Set dst con);
 7646 
 7647   ins_cost(INSN_COST);
 7648   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7649 
 7650   ins_encode(aarch64_enc_mov_n0(dst, con));
 7651 
 7652   ins_pipe(ialu_imm);
 7653 %}
 7654 
 7655 // Load Narrow Klass Constant
 7656 
 7657 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7658 %{
 7659   match(Set dst con);
 7660 
 7661   ins_cost(INSN_COST);
 7662   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7663 
 7664   ins_encode(aarch64_enc_mov_nk(dst, con));
 7665 
 7666   ins_pipe(ialu_imm);
 7667 %}
 7668 
 7669 // Load Packed Float Constant
 7670 
 7671 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7672   match(Set dst con);
 7673   ins_cost(INSN_COST * 4);
 7674   format %{ "fmovs  $dst, $con"%}
 7675   ins_encode %{
 7676     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7677   %}
 7678 
 7679   ins_pipe(fp_imm_s);
 7680 %}
 7681 
 7682 // Load Float Constant
 7683 
 7684 instruct loadConF(vRegF dst, immF con) %{
 7685   match(Set dst con);
 7686 
 7687   ins_cost(INSN_COST * 4);
 7688 
 7689   format %{
 7690     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7691   %}
 7692 
 7693   ins_encode %{
 7694     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7695   %}
 7696 
 7697   ins_pipe(fp_load_constant_s);
 7698 %}
 7699 
 7700 // Load Packed Double Constant
 7701 
 7702 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7703   match(Set dst con);
 7704   ins_cost(INSN_COST);
 7705   format %{ "fmovd  $dst, $con"%}
 7706   ins_encode %{
 7707     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7708   %}
 7709 
 7710   ins_pipe(fp_imm_d);
 7711 %}
 7712 
 7713 // Load Double Constant
 7714 
 7715 instruct loadConD(vRegD dst, immD con) %{
 7716   match(Set dst con);
 7717 
 7718   ins_cost(INSN_COST * 5);
 7719   format %{
 7720     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7721   %}
 7722 
 7723   ins_encode %{
 7724     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7725   %}
 7726 
 7727   ins_pipe(fp_load_constant_d);
 7728 %}
 7729 
 7730 // Store Instructions
 7731 
 7732 // Store CMS card-mark Immediate
 7733 instruct storeimmCM0(immI0 zero, memory1 mem)
 7734 %{
 7735   match(Set mem (StoreCM mem zero));
 7736 
 7737   ins_cost(INSN_COST);
 7738   format %{ "storestore (elided)\n\t"
 7739             "strb zr, $mem\t# byte" %}
 7740 
 7741   ins_encode(aarch64_enc_strb0(mem));
 7742 
 7743   ins_pipe(istore_mem);
 7744 %}
 7745 
 7746 // Store CMS card-mark Immediate with intervening StoreStore
 7747 // needed when using CMS with no conditional card marking
 7748 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7749 %{
 7750   match(Set mem (StoreCM mem zero));
 7751 
 7752   ins_cost(INSN_COST * 2);
 7753   format %{ "storestore\n\t"
 7754             "dmb ishst"
 7755             "\n\tstrb zr, $mem\t# byte" %}
 7756 
 7757   ins_encode(aarch64_enc_strb0_ordered(mem));
 7758 
 7759   ins_pipe(istore_mem);
 7760 %}
 7761 
 7762 // Store Byte
 7763 instruct storeB(iRegIorL2I src, memory1 mem)
 7764 %{
 7765   match(Set mem (StoreB mem src));
 7766   predicate(!needs_releasing_store(n));
 7767 
 7768   ins_cost(INSN_COST);
 7769   format %{ "strb  $src, $mem\t# byte" %}
 7770 
 7771   ins_encode(aarch64_enc_strb(src, mem));
 7772 
 7773   ins_pipe(istore_reg_mem);
 7774 %}
 7775 
 7776 
 7777 instruct storeimmB0(immI0 zero, memory1 mem)
 7778 %{
 7779   match(Set mem (StoreB mem zero));
 7780   predicate(!needs_releasing_store(n));
 7781 
 7782   ins_cost(INSN_COST);
 7783   format %{ "strb rscractch2, $mem\t# byte" %}
 7784 
 7785   ins_encode(aarch64_enc_strb0(mem));
 7786 
 7787   ins_pipe(istore_mem);
 7788 %}
 7789 
 7790 // Store Char/Short
 7791 instruct storeC(iRegIorL2I src, memory2 mem)
 7792 %{
 7793   match(Set mem (StoreC mem src));
 7794   predicate(!needs_releasing_store(n));
 7795 
 7796   ins_cost(INSN_COST);
 7797   format %{ "strh  $src, $mem\t# short" %}
 7798 
 7799   ins_encode(aarch64_enc_strh(src, mem));
 7800 
 7801   ins_pipe(istore_reg_mem);
 7802 %}
 7803 
 7804 instruct storeimmC0(immI0 zero, memory2 mem)
 7805 %{
 7806   match(Set mem (StoreC mem zero));
 7807   predicate(!needs_releasing_store(n));
 7808 
 7809   ins_cost(INSN_COST);
 7810   format %{ "strh  zr, $mem\t# short" %}
 7811 
 7812   ins_encode(aarch64_enc_strh0(mem));
 7813 
 7814   ins_pipe(istore_mem);
 7815 %}
 7816 
 7817 // Store Integer
 7818 
 7819 instruct storeI(iRegIorL2I src, memory4 mem)
 7820 %{
 7821   match(Set mem(StoreI mem src));
 7822   predicate(!needs_releasing_store(n));
 7823 
 7824   ins_cost(INSN_COST);
 7825   format %{ "strw  $src, $mem\t# int" %}
 7826 
 7827   ins_encode(aarch64_enc_strw(src, mem));
 7828 
 7829   ins_pipe(istore_reg_mem);
 7830 %}
 7831 
 7832 instruct storeimmI0(immI0 zero, memory4 mem)
 7833 %{
 7834   match(Set mem(StoreI mem zero));
 7835   predicate(!needs_releasing_store(n));
 7836 
 7837   ins_cost(INSN_COST);
 7838   format %{ "strw  zr, $mem\t# int" %}
 7839 
 7840   ins_encode(aarch64_enc_strw0(mem));
 7841 
 7842   ins_pipe(istore_mem);
 7843 %}
 7844 
 7845 // Store Long (64 bit signed)
 7846 instruct storeL(iRegL src, memory8 mem)
 7847 %{
 7848   match(Set mem (StoreL mem src));
 7849   predicate(!needs_releasing_store(n));
 7850 
 7851   ins_cost(INSN_COST);
 7852   format %{ "str  $src, $mem\t# int" %}
 7853 
 7854   ins_encode(aarch64_enc_str(src, mem));
 7855 
 7856   ins_pipe(istore_reg_mem);
 7857 %}
 7858 
 7859 // Store Long (64 bit signed)
 7860 instruct storeimmL0(immL0 zero, memory8 mem)
 7861 %{
 7862   match(Set mem (StoreL mem zero));
 7863   predicate(!needs_releasing_store(n));
 7864 
 7865   ins_cost(INSN_COST);
 7866   format %{ "str  zr, $mem\t# int" %}
 7867 
 7868   ins_encode(aarch64_enc_str0(mem));
 7869 
 7870   ins_pipe(istore_mem);
 7871 %}
 7872 
 7873 // Store Pointer
 7874 instruct storeP(iRegP src, memory8 mem)
 7875 %{
 7876   match(Set mem (StoreP mem src));
 7877   predicate(!needs_releasing_store(n));
 7878 
 7879   ins_cost(INSN_COST);
 7880   format %{ "str  $src, $mem\t# ptr" %}
 7881 
 7882   ins_encode(aarch64_enc_str(src, mem));
 7883 
 7884   ins_pipe(istore_reg_mem);
 7885 %}
 7886 
 7887 // Store Pointer
 7888 instruct storeimmP0(immP0 zero, memory8 mem)
 7889 %{
 7890   match(Set mem (StoreP mem zero));
 7891   predicate(!needs_releasing_store(n));
 7892 
 7893   ins_cost(INSN_COST);
 7894   format %{ "str zr, $mem\t# ptr" %}
 7895 
 7896   ins_encode(aarch64_enc_str0(mem));
 7897 
 7898   ins_pipe(istore_mem);
 7899 %}
 7900 
 7901 // Store Compressed Pointer
 7902 instruct storeN(iRegN src, memory4 mem)
 7903 %{
 7904   match(Set mem (StoreN mem src));
 7905   predicate(!needs_releasing_store(n));
 7906 
 7907   ins_cost(INSN_COST);
 7908   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7909 
 7910   ins_encode(aarch64_enc_strw(src, mem));
 7911 
 7912   ins_pipe(istore_reg_mem);
 7913 %}
 7914 
 7915 instruct storeImmN0(immN0 zero, memory4 mem)
 7916 %{
 7917   match(Set mem (StoreN mem zero));
 7918   predicate(!needs_releasing_store(n));
 7919 
 7920   ins_cost(INSN_COST);
 7921   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7922 
 7923   ins_encode(aarch64_enc_strw0(mem));
 7924 
 7925   ins_pipe(istore_mem);
 7926 %}
 7927 
 7928 // Store Float
 7929 instruct storeF(vRegF src, memory4 mem)
 7930 %{
 7931   match(Set mem (StoreF mem src));
 7932   predicate(!needs_releasing_store(n));
 7933 
 7934   ins_cost(INSN_COST);
 7935   format %{ "strs  $src, $mem\t# float" %}
 7936 
 7937   ins_encode( aarch64_enc_strs(src, mem) );
 7938 
 7939   ins_pipe(pipe_class_memory);
 7940 %}
 7941 
 7942 // TODO
 7943 // implement storeImmF0 and storeFImmPacked
 7944 
 7945 // Store Double
 7946 instruct storeD(vRegD src, memory8 mem)
 7947 %{
 7948   match(Set mem (StoreD mem src));
 7949   predicate(!needs_releasing_store(n));
 7950 
 7951   ins_cost(INSN_COST);
 7952   format %{ "strd  $src, $mem\t# double" %}
 7953 
 7954   ins_encode( aarch64_enc_strd(src, mem) );
 7955 
 7956   ins_pipe(pipe_class_memory);
 7957 %}
 7958 
 7959 // Store Compressed Klass Pointer
 7960 instruct storeNKlass(iRegN src, memory4 mem)
 7961 %{
 7962   predicate(!needs_releasing_store(n));
 7963   match(Set mem (StoreNKlass mem src));
 7964 
 7965   ins_cost(INSN_COST);
 7966   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7967 
 7968   ins_encode(aarch64_enc_strw(src, mem));
 7969 
 7970   ins_pipe(istore_reg_mem);
 7971 %}
 7972 
 7973 // TODO
 7974 // implement storeImmD0 and storeDImmPacked
 7975 
 7976 // prefetch instructions
 7977 // Must be safe to execute with invalid address (cannot fault).
 7978 
 7979 instruct prefetchalloc( memory8 mem ) %{
 7980   match(PrefetchAllocation mem);
 7981 
 7982   ins_cost(INSN_COST);
 7983   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7984 
 7985   ins_encode( aarch64_enc_prefetchw(mem) );
 7986 
 7987   ins_pipe(iload_prefetch);
 7988 %}
 7989 
 7990 //  ---------------- volatile loads and stores ----------------
 7991 
 7992 // Load Byte (8 bit signed)
 7993 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7994 %{
 7995   match(Set dst (LoadB mem));
 7996 
 7997   ins_cost(VOLATILE_REF_COST);
 7998   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7999 
 8000   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8001 
 8002   ins_pipe(pipe_serial);
 8003 %}
 8004 
 8005 // Load Byte (8 bit signed) into long
 8006 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8007 %{
 8008   match(Set dst (ConvI2L (LoadB mem)));
 8009 
 8010   ins_cost(VOLATILE_REF_COST);
 8011   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8012 
 8013   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8014 
 8015   ins_pipe(pipe_serial);
 8016 %}
 8017 
 8018 // Load Byte (8 bit unsigned)
 8019 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8020 %{
 8021   match(Set dst (LoadUB mem));
 8022 
 8023   ins_cost(VOLATILE_REF_COST);
 8024   format %{ "ldarb  $dst, $mem\t# byte" %}
 8025 
 8026   ins_encode(aarch64_enc_ldarb(dst, mem));
 8027 
 8028   ins_pipe(pipe_serial);
 8029 %}
 8030 
 8031 // Load Byte (8 bit unsigned) into long
 8032 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8033 %{
 8034   match(Set dst (ConvI2L (LoadUB mem)));
 8035 
 8036   ins_cost(VOLATILE_REF_COST);
 8037   format %{ "ldarb  $dst, $mem\t# byte" %}
 8038 
 8039   ins_encode(aarch64_enc_ldarb(dst, mem));
 8040 
 8041   ins_pipe(pipe_serial);
 8042 %}
 8043 
 8044 // Load Short (16 bit signed)
 8045 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8046 %{
 8047   match(Set dst (LoadS mem));
 8048 
 8049   ins_cost(VOLATILE_REF_COST);
 8050   format %{ "ldarshw  $dst, $mem\t# short" %}
 8051 
 8052   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8053 
 8054   ins_pipe(pipe_serial);
 8055 %}
 8056 
 8057 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8058 %{
 8059   match(Set dst (LoadUS mem));
 8060 
 8061   ins_cost(VOLATILE_REF_COST);
 8062   format %{ "ldarhw  $dst, $mem\t# short" %}
 8063 
 8064   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8065 
 8066   ins_pipe(pipe_serial);
 8067 %}
 8068 
 8069 // Load Short/Char (16 bit unsigned) into long
 8070 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8071 %{
 8072   match(Set dst (ConvI2L (LoadUS mem)));
 8073 
 8074   ins_cost(VOLATILE_REF_COST);
 8075   format %{ "ldarh  $dst, $mem\t# short" %}
 8076 
 8077   ins_encode(aarch64_enc_ldarh(dst, mem));
 8078 
 8079   ins_pipe(pipe_serial);
 8080 %}
 8081 
 8082 // Load Short/Char (16 bit signed) into long
 8083 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8084 %{
 8085   match(Set dst (ConvI2L (LoadS mem)));
 8086 
 8087   ins_cost(VOLATILE_REF_COST);
 8088   format %{ "ldarh  $dst, $mem\t# short" %}
 8089 
 8090   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8091 
 8092   ins_pipe(pipe_serial);
 8093 %}
 8094 
 8095 // Load Integer (32 bit signed)
 8096 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8097 %{
 8098   match(Set dst (LoadI mem));
 8099 
 8100   ins_cost(VOLATILE_REF_COST);
 8101   format %{ "ldarw  $dst, $mem\t# int" %}
 8102 
 8103   ins_encode(aarch64_enc_ldarw(dst, mem));
 8104 
 8105   ins_pipe(pipe_serial);
 8106 %}
 8107 
 8108 // Load Integer (32 bit unsigned) into long
 8109 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8110 %{
 8111   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8112 
 8113   ins_cost(VOLATILE_REF_COST);
 8114   format %{ "ldarw  $dst, $mem\t# int" %}
 8115 
 8116   ins_encode(aarch64_enc_ldarw(dst, mem));
 8117 
 8118   ins_pipe(pipe_serial);
 8119 %}
 8120 
 8121 // Load Long (64 bit signed)
 8122 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8123 %{
 8124   match(Set dst (LoadL mem));
 8125 
 8126   ins_cost(VOLATILE_REF_COST);
 8127   format %{ "ldar  $dst, $mem\t# int" %}
 8128 
 8129   ins_encode(aarch64_enc_ldar(dst, mem));
 8130 
 8131   ins_pipe(pipe_serial);
 8132 %}
 8133 
 8134 // Load Pointer
 8135 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8136 %{
 8137   match(Set dst (LoadP mem));
 8138   predicate(n->as_Load()->barrier_data() == 0);
 8139 
 8140   ins_cost(VOLATILE_REF_COST);
 8141   format %{ "ldar  $dst, $mem\t# ptr" %}
 8142 
 8143   ins_encode(aarch64_enc_ldar(dst, mem));
 8144 
 8145   ins_pipe(pipe_serial);
 8146 %}
 8147 
 8148 // Load Compressed Pointer
 8149 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8150 %{
 8151   match(Set dst (LoadN mem));
 8152 
 8153   ins_cost(VOLATILE_REF_COST);
 8154   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8155 
 8156   ins_encode(aarch64_enc_ldarw(dst, mem));
 8157 
 8158   ins_pipe(pipe_serial);
 8159 %}
 8160 
 8161 // Load Float
 8162 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8163 %{
 8164   match(Set dst (LoadF mem));
 8165 
 8166   ins_cost(VOLATILE_REF_COST);
 8167   format %{ "ldars  $dst, $mem\t# float" %}
 8168 
 8169   ins_encode( aarch64_enc_fldars(dst, mem) );
 8170 
 8171   ins_pipe(pipe_serial);
 8172 %}
 8173 
 8174 // Load Double
 8175 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8176 %{
 8177   match(Set dst (LoadD mem));
 8178 
 8179   ins_cost(VOLATILE_REF_COST);
 8180   format %{ "ldard  $dst, $mem\t# double" %}
 8181 
 8182   ins_encode( aarch64_enc_fldard(dst, mem) );
 8183 
 8184   ins_pipe(pipe_serial);
 8185 %}
 8186 
 8187 // Store Byte
 8188 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8189 %{
 8190   match(Set mem (StoreB mem src));
 8191 
 8192   ins_cost(VOLATILE_REF_COST);
 8193   format %{ "stlrb  $src, $mem\t# byte" %}
 8194 
 8195   ins_encode(aarch64_enc_stlrb(src, mem));
 8196 
 8197   ins_pipe(pipe_class_memory);
 8198 %}
 8199 
 8200 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8201 %{
 8202   match(Set mem (StoreB mem zero));
 8203 
 8204   ins_cost(VOLATILE_REF_COST);
 8205   format %{ "stlrb  zr, $mem\t# byte" %}
 8206 
 8207   ins_encode(aarch64_enc_stlrb0(mem));
 8208 
 8209   ins_pipe(pipe_class_memory);
 8210 %}
 8211 
 8212 // Store Char/Short
 8213 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8214 %{
 8215   match(Set mem (StoreC mem src));
 8216 
 8217   ins_cost(VOLATILE_REF_COST);
 8218   format %{ "stlrh  $src, $mem\t# short" %}
 8219 
 8220   ins_encode(aarch64_enc_stlrh(src, mem));
 8221 
 8222   ins_pipe(pipe_class_memory);
 8223 %}
 8224 
 8225 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8226 %{
 8227   match(Set mem (StoreC mem zero));
 8228 
 8229   ins_cost(VOLATILE_REF_COST);
 8230   format %{ "stlrh  zr, $mem\t# short" %}
 8231 
 8232   ins_encode(aarch64_enc_stlrh0(mem));
 8233 
 8234   ins_pipe(pipe_class_memory);
 8235 %}
 8236 
 8237 // Store Integer
 8238 
 8239 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8240 %{
 8241   match(Set mem(StoreI mem src));
 8242 
 8243   ins_cost(VOLATILE_REF_COST);
 8244   format %{ "stlrw  $src, $mem\t# int" %}
 8245 
 8246   ins_encode(aarch64_enc_stlrw(src, mem));
 8247 
 8248   ins_pipe(pipe_class_memory);
 8249 %}
 8250 
 8251 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8252 %{
 8253   match(Set mem(StoreI mem zero));
 8254 
 8255   ins_cost(VOLATILE_REF_COST);
 8256   format %{ "stlrw  zr, $mem\t# int" %}
 8257 
 8258   ins_encode(aarch64_enc_stlrw0(mem));
 8259 
 8260   ins_pipe(pipe_class_memory);
 8261 %}
 8262 
 8263 // Store Long (64 bit signed)
 8264 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8265 %{
 8266   match(Set mem (StoreL mem src));
 8267 
 8268   ins_cost(VOLATILE_REF_COST);
 8269   format %{ "stlr  $src, $mem\t# int" %}
 8270 
 8271   ins_encode(aarch64_enc_stlr(src, mem));
 8272 
 8273   ins_pipe(pipe_class_memory);
 8274 %}
 8275 
 8276 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8277 %{
 8278   match(Set mem (StoreL mem zero));
 8279 
 8280   ins_cost(VOLATILE_REF_COST);
 8281   format %{ "stlr  zr, $mem\t# int" %}
 8282 
 8283   ins_encode(aarch64_enc_stlr0(mem));
 8284 
 8285   ins_pipe(pipe_class_memory);
 8286 %}
 8287 
 8288 // Store Pointer
 8289 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8290 %{
 8291   match(Set mem (StoreP mem src));
 8292 
 8293   ins_cost(VOLATILE_REF_COST);
 8294   format %{ "stlr  $src, $mem\t# ptr" %}
 8295 
 8296   ins_encode(aarch64_enc_stlr(src, mem));
 8297 
 8298   ins_pipe(pipe_class_memory);
 8299 %}
 8300 
 8301 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8302 %{
 8303   match(Set mem (StoreP mem zero));
 8304 
 8305   ins_cost(VOLATILE_REF_COST);
 8306   format %{ "stlr  zr, $mem\t# ptr" %}
 8307 
 8308   ins_encode(aarch64_enc_stlr0(mem));
 8309 
 8310   ins_pipe(pipe_class_memory);
 8311 %}
 8312 
 8313 // Store Compressed Pointer
 8314 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8315 %{
 8316   match(Set mem (StoreN mem src));
 8317 
 8318   ins_cost(VOLATILE_REF_COST);
 8319   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8320 
 8321   ins_encode(aarch64_enc_stlrw(src, mem));
 8322 
 8323   ins_pipe(pipe_class_memory);
 8324 %}
 8325 
 8326 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8327 %{
 8328   match(Set mem (StoreN mem zero));
 8329 
 8330   ins_cost(VOLATILE_REF_COST);
 8331   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8332 
 8333   ins_encode(aarch64_enc_stlrw0(mem));
 8334 
 8335   ins_pipe(pipe_class_memory);
 8336 %}
 8337 
 8338 // Store Float
 8339 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8340 %{
 8341   match(Set mem (StoreF mem src));
 8342 
 8343   ins_cost(VOLATILE_REF_COST);
 8344   format %{ "stlrs  $src, $mem\t# float" %}
 8345 
 8346   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8347 
 8348   ins_pipe(pipe_class_memory);
 8349 %}
 8350 
 8351 // TODO
 8352 // implement storeImmF0 and storeFImmPacked
 8353 
 8354 // Store Double
 8355 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8356 %{
 8357   match(Set mem (StoreD mem src));
 8358 
 8359   ins_cost(VOLATILE_REF_COST);
 8360   format %{ "stlrd  $src, $mem\t# double" %}
 8361 
 8362   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8363 
 8364   ins_pipe(pipe_class_memory);
 8365 %}
 8366 
 8367 //  ---------------- end of volatile loads and stores ----------------
 8368 
 8369 instruct cacheWB(indirect addr)
 8370 %{
 8371   predicate(VM_Version::supports_data_cache_line_flush());
 8372   match(CacheWB addr);
 8373 
 8374   ins_cost(100);
 8375   format %{"cache wb $addr" %}
 8376   ins_encode %{
 8377     assert($addr->index_position() < 0, "should be");
 8378     assert($addr$$disp == 0, "should be");
 8379     __ cache_wb(Address($addr$$base$$Register, 0));
 8380   %}
 8381   ins_pipe(pipe_slow); // XXX
 8382 %}
 8383 
 8384 instruct cacheWBPreSync()
 8385 %{
 8386   predicate(VM_Version::supports_data_cache_line_flush());
 8387   match(CacheWBPreSync);
 8388 
 8389   ins_cost(100);
 8390   format %{"cache wb presync" %}
 8391   ins_encode %{
 8392     __ cache_wbsync(true);
 8393   %}
 8394   ins_pipe(pipe_slow); // XXX
 8395 %}
 8396 
 8397 instruct cacheWBPostSync()
 8398 %{
 8399   predicate(VM_Version::supports_data_cache_line_flush());
 8400   match(CacheWBPostSync);
 8401 
 8402   ins_cost(100);
 8403   format %{"cache wb postsync" %}
 8404   ins_encode %{
 8405     __ cache_wbsync(false);
 8406   %}
 8407   ins_pipe(pipe_slow); // XXX
 8408 %}
 8409 
 8410 // ============================================================================
 8411 // BSWAP Instructions
 8412 
 8413 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8414   match(Set dst (ReverseBytesI src));
 8415 
 8416   ins_cost(INSN_COST);
 8417   format %{ "revw  $dst, $src" %}
 8418 
 8419   ins_encode %{
 8420     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8421   %}
 8422 
 8423   ins_pipe(ialu_reg);
 8424 %}
 8425 
 8426 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8427   match(Set dst (ReverseBytesL src));
 8428 
 8429   ins_cost(INSN_COST);
 8430   format %{ "rev  $dst, $src" %}
 8431 
 8432   ins_encode %{
 8433     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8434   %}
 8435 
 8436   ins_pipe(ialu_reg);
 8437 %}
 8438 
 8439 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8440   match(Set dst (ReverseBytesUS src));
 8441 
 8442   ins_cost(INSN_COST);
 8443   format %{ "rev16w  $dst, $src" %}
 8444 
 8445   ins_encode %{
 8446     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8447   %}
 8448 
 8449   ins_pipe(ialu_reg);
 8450 %}
 8451 
 8452 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8453   match(Set dst (ReverseBytesS src));
 8454 
 8455   ins_cost(INSN_COST);
 8456   format %{ "rev16w  $dst, $src\n\t"
 8457             "sbfmw $dst, $dst, #0, #15" %}
 8458 
 8459   ins_encode %{
 8460     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8461     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8462   %}
 8463 
 8464   ins_pipe(ialu_reg);
 8465 %}
 8466 
 8467 // ============================================================================
 8468 // Zero Count Instructions
 8469 
 8470 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8471   match(Set dst (CountLeadingZerosI src));
 8472 
 8473   ins_cost(INSN_COST);
 8474   format %{ "clzw  $dst, $src" %}
 8475   ins_encode %{
 8476     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8477   %}
 8478 
 8479   ins_pipe(ialu_reg);
 8480 %}
 8481 
 8482 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8483   match(Set dst (CountLeadingZerosL src));
 8484 
 8485   ins_cost(INSN_COST);
 8486   format %{ "clz   $dst, $src" %}
 8487   ins_encode %{
 8488     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8489   %}
 8490 
 8491   ins_pipe(ialu_reg);
 8492 %}
 8493 
 8494 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8495   match(Set dst (CountTrailingZerosI src));
 8496 
 8497   ins_cost(INSN_COST * 2);
 8498   format %{ "rbitw  $dst, $src\n\t"
 8499             "clzw   $dst, $dst" %}
 8500   ins_encode %{
 8501     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8502     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8503   %}
 8504 
 8505   ins_pipe(ialu_reg);
 8506 %}
 8507 
 8508 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8509   match(Set dst (CountTrailingZerosL src));
 8510 
 8511   ins_cost(INSN_COST * 2);
 8512   format %{ "rbit   $dst, $src\n\t"
 8513             "clz    $dst, $dst" %}
 8514   ins_encode %{
 8515     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8516     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8517   %}
 8518 
 8519   ins_pipe(ialu_reg);
 8520 %}
 8521 
 8522 //---------- Population Count Instructions -------------------------------------
 8523 //
 8524 
 8525 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8526   predicate(UsePopCountInstruction);
 8527   match(Set dst (PopCountI src));
 8528   effect(TEMP tmp);
 8529   ins_cost(INSN_COST * 13);
 8530 
 8531   format %{ "movw   $src, $src\n\t"
 8532             "mov    $tmp, $src\t# vector (1D)\n\t"
 8533             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8534             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8535             "mov    $dst, $tmp\t# vector (1D)" %}
 8536   ins_encode %{
 8537     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8538     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8539     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8540     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8541     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8542   %}
 8543 
 8544   ins_pipe(pipe_class_default);
 8545 %}
 8546 
 8547 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8548   predicate(UsePopCountInstruction);
 8549   match(Set dst (PopCountI (LoadI mem)));
 8550   effect(TEMP tmp);
 8551   ins_cost(INSN_COST * 13);
 8552 
 8553   format %{ "ldrs   $tmp, $mem\n\t"
 8554             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8555             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8556             "mov    $dst, $tmp\t# vector (1D)" %}
 8557   ins_encode %{
 8558     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8559     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8560               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8561     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8562     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8563     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8564   %}
 8565 
 8566   ins_pipe(pipe_class_default);
 8567 %}
 8568 
 8569 // Note: Long.bitCount(long) returns an int.
 8570 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8571   predicate(UsePopCountInstruction);
 8572   match(Set dst (PopCountL src));
 8573   effect(TEMP tmp);
 8574   ins_cost(INSN_COST * 13);
 8575 
 8576   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8577             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8578             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8579             "mov    $dst, $tmp\t# vector (1D)" %}
 8580   ins_encode %{
 8581     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8582     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8583     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8584     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8585   %}
 8586 
 8587   ins_pipe(pipe_class_default);
 8588 %}
 8589 
 8590 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8591   predicate(UsePopCountInstruction);
 8592   match(Set dst (PopCountL (LoadL mem)));
 8593   effect(TEMP tmp);
 8594   ins_cost(INSN_COST * 13);
 8595 
 8596   format %{ "ldrd   $tmp, $mem\n\t"
 8597             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8598             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8599             "mov    $dst, $tmp\t# vector (1D)" %}
 8600   ins_encode %{
 8601     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8602     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8603               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8604     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8605     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8606     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8607   %}
 8608 
 8609   ins_pipe(pipe_class_default);
 8610 %}
 8611 
 8612 // ============================================================================
 8613 // MemBar Instruction
 8614 
 8615 instruct load_fence() %{
 8616   match(LoadFence);
 8617   ins_cost(VOLATILE_REF_COST);
 8618 
 8619   format %{ "load_fence" %}
 8620 
 8621   ins_encode %{
 8622     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8623   %}
 8624   ins_pipe(pipe_serial);
 8625 %}
 8626 
 8627 instruct unnecessary_membar_acquire() %{
 8628   predicate(unnecessary_acquire(n));
 8629   match(MemBarAcquire);
 8630   ins_cost(0);
 8631 
 8632   format %{ "membar_acquire (elided)" %}
 8633 
 8634   ins_encode %{
 8635     __ block_comment("membar_acquire (elided)");
 8636   %}
 8637 
 8638   ins_pipe(pipe_class_empty);
 8639 %}
 8640 
 8641 instruct membar_acquire() %{
 8642   match(MemBarAcquire);
 8643   ins_cost(VOLATILE_REF_COST);
 8644 
 8645   format %{ "membar_acquire\n\t"
 8646             "dmb ish" %}
 8647 
 8648   ins_encode %{
 8649     __ block_comment("membar_acquire");
 8650     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8651   %}
 8652 
 8653   ins_pipe(pipe_serial);
 8654 %}
 8655 
 8656 
 8657 instruct membar_acquire_lock() %{
 8658   match(MemBarAcquireLock);
 8659   ins_cost(VOLATILE_REF_COST);
 8660 
 8661   format %{ "membar_acquire_lock (elided)" %}
 8662 
 8663   ins_encode %{
 8664     __ block_comment("membar_acquire_lock (elided)");
 8665   %}
 8666 
 8667   ins_pipe(pipe_serial);
 8668 %}
 8669 
 8670 instruct store_fence() %{
 8671   match(StoreFence);
 8672   ins_cost(VOLATILE_REF_COST);
 8673 
 8674   format %{ "store_fence" %}
 8675 
 8676   ins_encode %{
 8677     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8678   %}
 8679   ins_pipe(pipe_serial);
 8680 %}
 8681 
 8682 instruct unnecessary_membar_release() %{
 8683   predicate(unnecessary_release(n));
 8684   match(MemBarRelease);
 8685   ins_cost(0);
 8686 
 8687   format %{ "membar_release (elided)" %}
 8688 
 8689   ins_encode %{
 8690     __ block_comment("membar_release (elided)");
 8691   %}
 8692   ins_pipe(pipe_serial);
 8693 %}
 8694 
 8695 instruct membar_release() %{
 8696   match(MemBarRelease);
 8697   ins_cost(VOLATILE_REF_COST);
 8698 
 8699   format %{ "membar_release\n\t"
 8700             "dmb ish" %}
 8701 
 8702   ins_encode %{
 8703     __ block_comment("membar_release");
 8704     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8705   %}
 8706   ins_pipe(pipe_serial);
 8707 %}
 8708 
 8709 instruct membar_storestore() %{
 8710   match(MemBarStoreStore);
 8711   match(StoreStoreFence);
 8712   ins_cost(VOLATILE_REF_COST);
 8713 
 8714   format %{ "MEMBAR-store-store" %}
 8715 
 8716   ins_encode %{
 8717     __ membar(Assembler::StoreStore);
 8718   %}
 8719   ins_pipe(pipe_serial);
 8720 %}
 8721 
 8722 instruct membar_release_lock() %{
 8723   match(MemBarReleaseLock);
 8724   ins_cost(VOLATILE_REF_COST);
 8725 
 8726   format %{ "membar_release_lock (elided)" %}
 8727 
 8728   ins_encode %{
 8729     __ block_comment("membar_release_lock (elided)");
 8730   %}
 8731 
 8732   ins_pipe(pipe_serial);
 8733 %}
 8734 
 8735 instruct unnecessary_membar_volatile() %{
 8736   predicate(unnecessary_volatile(n));
 8737   match(MemBarVolatile);
 8738   ins_cost(0);
 8739 
 8740   format %{ "membar_volatile (elided)" %}
 8741 
 8742   ins_encode %{
 8743     __ block_comment("membar_volatile (elided)");
 8744   %}
 8745 
 8746   ins_pipe(pipe_serial);
 8747 %}
 8748 
 8749 instruct membar_volatile() %{
 8750   match(MemBarVolatile);
 8751   ins_cost(VOLATILE_REF_COST*100);
 8752 
 8753   format %{ "membar_volatile\n\t"
 8754              "dmb ish"%}
 8755 
 8756   ins_encode %{
 8757     __ block_comment("membar_volatile");
 8758     __ membar(Assembler::StoreLoad);
 8759   %}
 8760 
 8761   ins_pipe(pipe_serial);
 8762 %}
 8763 
 8764 // ============================================================================
 8765 // Cast/Convert Instructions
 8766 
 8767 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8768   match(Set dst (CastX2P src));
 8769 
 8770   ins_cost(INSN_COST);
 8771   format %{ "mov $dst, $src\t# long -> ptr" %}
 8772 
 8773   ins_encode %{
 8774     if ($dst$$reg != $src$$reg) {
 8775       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8776     }
 8777   %}
 8778 
 8779   ins_pipe(ialu_reg);
 8780 %}
 8781 
 8782 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8783   match(Set dst (CastP2X src));
 8784 
 8785   ins_cost(INSN_COST);
 8786   format %{ "mov $dst, $src\t# ptr -> long" %}
 8787 
 8788   ins_encode %{
 8789     if ($dst$$reg != $src$$reg) {
 8790       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8791     }
 8792   %}
 8793 
 8794   ins_pipe(ialu_reg);
 8795 %}
 8796 
 8797 // Convert oop into int for vectors alignment masking
 8798 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8799   match(Set dst (ConvL2I (CastP2X src)));
 8800 
 8801   ins_cost(INSN_COST);
 8802   format %{ "movw $dst, $src\t# ptr -> int" %}
 8803   ins_encode %{
 8804     __ movw($dst$$Register, $src$$Register);
 8805   %}
 8806 
 8807   ins_pipe(ialu_reg);
 8808 %}
 8809 
 8810 // Convert compressed oop into int for vectors alignment masking
 8811 // in case of 32bit oops (heap < 4Gb).
 8812 instruct convN2I(iRegINoSp dst, iRegN src)
 8813 %{
 8814   predicate(CompressedOops::shift() == 0);
 8815   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8816 
 8817   ins_cost(INSN_COST);
 8818   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8819   ins_encode %{
 8820     __ movw($dst$$Register, $src$$Register);
 8821   %}
 8822 
 8823   ins_pipe(ialu_reg);
 8824 %}
 8825 
 8826 
 8827 // Convert oop pointer into compressed form
 8828 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8829   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8830   match(Set dst (EncodeP src));
 8831   effect(KILL cr);
 8832   ins_cost(INSN_COST * 3);
 8833   format %{ "encode_heap_oop $dst, $src" %}
 8834   ins_encode %{
 8835     Register s = $src$$Register;
 8836     Register d = $dst$$Register;
 8837     __ encode_heap_oop(d, s);
 8838   %}
 8839   ins_pipe(ialu_reg);
 8840 %}
 8841 
 8842 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8843   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8844   match(Set dst (EncodeP src));
 8845   ins_cost(INSN_COST * 3);
 8846   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8847   ins_encode %{
 8848     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8849   %}
 8850   ins_pipe(ialu_reg);
 8851 %}
 8852 
 8853 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8854   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8855             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8856   match(Set dst (DecodeN src));
 8857   ins_cost(INSN_COST * 3);
 8858   format %{ "decode_heap_oop $dst, $src" %}
 8859   ins_encode %{
 8860     Register s = $src$$Register;
 8861     Register d = $dst$$Register;
 8862     __ decode_heap_oop(d, s);
 8863   %}
 8864   ins_pipe(ialu_reg);
 8865 %}
 8866 
 8867 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8868   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8869             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8870   match(Set dst (DecodeN src));
 8871   ins_cost(INSN_COST * 3);
 8872   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8873   ins_encode %{
 8874     Register s = $src$$Register;
 8875     Register d = $dst$$Register;
 8876     __ decode_heap_oop_not_null(d, s);
 8877   %}
 8878   ins_pipe(ialu_reg);
 8879 %}
 8880 
 8881 // n.b. AArch64 implementations of encode_klass_not_null and
 8882 // decode_klass_not_null do not modify the flags register so, unlike
 8883 // Intel, we don't kill CR as a side effect here
 8884 
 8885 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8886   match(Set dst (EncodePKlass src));
 8887 
 8888   ins_cost(INSN_COST * 3);
 8889   format %{ "encode_klass_not_null $dst,$src" %}
 8890 
 8891   ins_encode %{
 8892     Register src_reg = as_Register($src$$reg);
 8893     Register dst_reg = as_Register($dst$$reg);
 8894     __ encode_klass_not_null(dst_reg, src_reg);
 8895   %}
 8896 
 8897    ins_pipe(ialu_reg);
 8898 %}
 8899 
 8900 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8901   match(Set dst (DecodeNKlass src));
 8902 
 8903   ins_cost(INSN_COST * 3);
 8904   format %{ "decode_klass_not_null $dst,$src" %}
 8905 
 8906   ins_encode %{
 8907     Register src_reg = as_Register($src$$reg);
 8908     Register dst_reg = as_Register($dst$$reg);
 8909     if (dst_reg != src_reg) {
 8910       __ decode_klass_not_null(dst_reg, src_reg);
 8911     } else {
 8912       __ decode_klass_not_null(dst_reg);
 8913     }
 8914   %}
 8915 
 8916    ins_pipe(ialu_reg);
 8917 %}
 8918 
 8919 instruct checkCastPP(iRegPNoSp dst)
 8920 %{
 8921   match(Set dst (CheckCastPP dst));
 8922 
 8923   size(0);
 8924   format %{ "# checkcastPP of $dst" %}
 8925   ins_encode(/* empty encoding */);
 8926   ins_pipe(pipe_class_empty);
 8927 %}
 8928 
 8929 instruct castPP(iRegPNoSp dst)
 8930 %{
 8931   match(Set dst (CastPP dst));
 8932 
 8933   size(0);
 8934   format %{ "# castPP of $dst" %}
 8935   ins_encode(/* empty encoding */);
 8936   ins_pipe(pipe_class_empty);
 8937 %}
 8938 
 8939 instruct castII(iRegI dst)
 8940 %{
 8941   match(Set dst (CastII dst));
 8942 
 8943   size(0);
 8944   format %{ "# castII of $dst" %}
 8945   ins_encode(/* empty encoding */);
 8946   ins_cost(0);
 8947   ins_pipe(pipe_class_empty);
 8948 %}
 8949 
 8950 instruct castLL(iRegL dst)
 8951 %{
 8952   match(Set dst (CastLL dst));
 8953 
 8954   size(0);
 8955   format %{ "# castLL of $dst" %}
 8956   ins_encode(/* empty encoding */);
 8957   ins_cost(0);
 8958   ins_pipe(pipe_class_empty);
 8959 %}
 8960 
 8961 instruct castFF(vRegF dst)
 8962 %{
 8963   match(Set dst (CastFF dst));
 8964 
 8965   size(0);
 8966   format %{ "# castFF of $dst" %}
 8967   ins_encode(/* empty encoding */);
 8968   ins_cost(0);
 8969   ins_pipe(pipe_class_empty);
 8970 %}
 8971 
 8972 instruct castDD(vRegD dst)
 8973 %{
 8974   match(Set dst (CastDD dst));
 8975 
 8976   size(0);
 8977   format %{ "# castDD of $dst" %}
 8978   ins_encode(/* empty encoding */);
 8979   ins_cost(0);
 8980   ins_pipe(pipe_class_empty);
 8981 %}
 8982 
 8983 instruct castVVD(vecD dst)
 8984 %{
 8985   match(Set dst (CastVV dst));
 8986 
 8987   size(0);
 8988   format %{ "# castVV of $dst" %}
 8989   ins_encode(/* empty encoding */);
 8990   ins_cost(0);
 8991   ins_pipe(pipe_class_empty);
 8992 %}
 8993 
 8994 instruct castVVX(vecX dst)
 8995 %{
 8996   match(Set dst (CastVV dst));
 8997 
 8998   size(0);
 8999   format %{ "# castVV of $dst" %}
 9000   ins_encode(/* empty encoding */);
 9001   ins_cost(0);
 9002   ins_pipe(pipe_class_empty);
 9003 %}
 9004 
 9005 instruct castVV(vReg dst)
 9006 %{
 9007   match(Set dst (CastVV dst));
 9008 
 9009   size(0);
 9010   format %{ "# castVV of $dst" %}
 9011   ins_encode(/* empty encoding */);
 9012   ins_cost(0);
 9013   ins_pipe(pipe_class_empty);
 9014 %}
 9015 
 9016 // ============================================================================
 9017 // Atomic operation instructions
 9018 //
 9019 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9020 // Store{PIL}Conditional instructions using a normal load for the
 9021 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9022 //
 9023 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9024 // pair to lock object allocations from Eden space when not using
 9025 // TLABs.
 9026 //
 9027 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9028 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9029 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9030 // only for 64-bit.
 9031 //
 9032 // We implement LoadPLocked and StorePLocked instructions using,
 9033 // respectively the AArch64 hw load-exclusive and store-conditional
 9034 // instructions. Whereas we must implement each of
 9035 // Store{IL}Conditional using a CAS which employs a pair of
 9036 // instructions comprising a load-exclusive followed by a
 9037 // store-conditional.
 9038 
 9039 
 9040 // Locked-load (linked load) of the current heap-top
 9041 // used when updating the eden heap top
 9042 // implemented using ldaxr on AArch64
 9043 
 9044 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9045 %{
 9046   match(Set dst (LoadPLocked mem));
 9047 
 9048   ins_cost(VOLATILE_REF_COST);
 9049 
 9050   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9051 
 9052   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9053 
 9054   ins_pipe(pipe_serial);
 9055 %}
 9056 
 9057 // Conditional-store of the updated heap-top.
 9058 // Used during allocation of the shared heap.
 9059 // Sets flag (EQ) on success.
 9060 // implemented using stlxr on AArch64.
 9061 
 9062 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9063 %{
 9064   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9065 
 9066   ins_cost(VOLATILE_REF_COST);
 9067 
 9068  // TODO
 9069  // do we need to do a store-conditional release or can we just use a
 9070  // plain store-conditional?
 9071 
 9072   format %{
 9073     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9074     "cmpw rscratch1, zr\t# EQ on successful write"
 9075   %}
 9076 
 9077   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9078 
 9079   ins_pipe(pipe_serial);
 9080 %}
 9081 
 9082 
 9083 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
 9084 // when attempting to rebias a lock towards the current thread.  We
 9085 // must use the acquire form of cmpxchg in order to guarantee acquire
 9086 // semantics in this case.
 9087 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9088 %{
 9089   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9090 
 9091   ins_cost(VOLATILE_REF_COST);
 9092 
 9093   format %{
 9094     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9095     "cmpw rscratch1, zr\t# EQ on successful write"
 9096   %}
 9097 
 9098   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9099 
 9100   ins_pipe(pipe_slow);
 9101 %}
 9102 
 9103 // storeIConditional also has acquire semantics, for no better reason
 9104 // than matching storeLConditional.  At the time of writing this
 9105 // comment storeIConditional was not used anywhere by AArch64.
 9106 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9107 %{
 9108   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9109 
 9110   ins_cost(VOLATILE_REF_COST);
 9111 
 9112   format %{
 9113     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9114     "cmpw rscratch1, zr\t# EQ on successful write"
 9115   %}
 9116 
 9117   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9118 
 9119   ins_pipe(pipe_slow);
 9120 %}
 9121 
 9122 // standard CompareAndSwapX when we are using barriers
 9123 // these have higher priority than the rules selected by a predicate
 9124 
 9125 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9126 // can't match them
 9127 
 9128 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9129 
 9130   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9131   ins_cost(2 * VOLATILE_REF_COST);
 9132 
 9133   effect(KILL cr);
 9134 
 9135   format %{
 9136     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9137     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9138   %}
 9139 
 9140   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9141             aarch64_enc_cset_eq(res));
 9142 
 9143   ins_pipe(pipe_slow);
 9144 %}
 9145 
 9146 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9147 
 9148   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9149   ins_cost(2 * VOLATILE_REF_COST);
 9150 
 9151   effect(KILL cr);
 9152 
 9153   format %{
 9154     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9155     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9156   %}
 9157 
 9158   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9159             aarch64_enc_cset_eq(res));
 9160 
 9161   ins_pipe(pipe_slow);
 9162 %}
 9163 
 9164 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9165 
 9166   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9167   ins_cost(2 * VOLATILE_REF_COST);
 9168 
 9169   effect(KILL cr);
 9170 
 9171  format %{
 9172     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9173     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9174  %}
 9175 
 9176  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9177             aarch64_enc_cset_eq(res));
 9178 
 9179   ins_pipe(pipe_slow);
 9180 %}
 9181 
 9182 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9183 
 9184   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9185   ins_cost(2 * VOLATILE_REF_COST);
 9186 
 9187   effect(KILL cr);
 9188 
 9189  format %{
 9190     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9191     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9192  %}
 9193 
 9194  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9195             aarch64_enc_cset_eq(res));
 9196 
 9197   ins_pipe(pipe_slow);
 9198 %}
 9199 
 9200 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9201 
 9202   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9203   predicate(n->as_LoadStore()->barrier_data() == 0);
 9204   ins_cost(2 * VOLATILE_REF_COST);
 9205 
 9206   effect(KILL cr);
 9207 
 9208  format %{
 9209     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9210     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9211  %}
 9212 
 9213  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9214             aarch64_enc_cset_eq(res));
 9215 
 9216   ins_pipe(pipe_slow);
 9217 %}
 9218 
 9219 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9220 
 9221   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9222   ins_cost(2 * VOLATILE_REF_COST);
 9223 
 9224   effect(KILL cr);
 9225 
 9226  format %{
 9227     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9228     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9229  %}
 9230 
 9231  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9232             aarch64_enc_cset_eq(res));
 9233 
 9234   ins_pipe(pipe_slow);
 9235 %}
 9236 
 9237 // alternative CompareAndSwapX when we are eliding barriers
 9238 
 9239 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9240 
 9241   predicate(needs_acquiring_load_exclusive(n));
 9242   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9243   ins_cost(VOLATILE_REF_COST);
 9244 
 9245   effect(KILL cr);
 9246 
 9247   format %{
 9248     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9249     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9250   %}
 9251 
 9252   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9253             aarch64_enc_cset_eq(res));
 9254 
 9255   ins_pipe(pipe_slow);
 9256 %}
 9257 
 9258 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9259 
 9260   predicate(needs_acquiring_load_exclusive(n));
 9261   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9262   ins_cost(VOLATILE_REF_COST);
 9263 
 9264   effect(KILL cr);
 9265 
 9266   format %{
 9267     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9268     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9269   %}
 9270 
 9271   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9272             aarch64_enc_cset_eq(res));
 9273 
 9274   ins_pipe(pipe_slow);
 9275 %}
 9276 
 9277 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9278 
 9279   predicate(needs_acquiring_load_exclusive(n));
 9280   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9281   ins_cost(VOLATILE_REF_COST);
 9282 
 9283   effect(KILL cr);
 9284 
 9285  format %{
 9286     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9287     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9288  %}
 9289 
 9290  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9291             aarch64_enc_cset_eq(res));
 9292 
 9293   ins_pipe(pipe_slow);
 9294 %}
 9295 
 9296 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9297 
 9298   predicate(needs_acquiring_load_exclusive(n));
 9299   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9300   ins_cost(VOLATILE_REF_COST);
 9301 
 9302   effect(KILL cr);
 9303 
 9304  format %{
 9305     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9306     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9307  %}
 9308 
 9309  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9310             aarch64_enc_cset_eq(res));
 9311 
 9312   ins_pipe(pipe_slow);
 9313 %}
 9314 
 9315 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9316 
 9317   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9318   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9319   ins_cost(VOLATILE_REF_COST);
 9320 
 9321   effect(KILL cr);
 9322 
 9323  format %{
 9324     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9325     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9326  %}
 9327 
 9328  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9329             aarch64_enc_cset_eq(res));
 9330 
 9331   ins_pipe(pipe_slow);
 9332 %}
 9333 
 9334 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9335 
 9336   predicate(needs_acquiring_load_exclusive(n));
 9337   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9338   ins_cost(VOLATILE_REF_COST);
 9339 
 9340   effect(KILL cr);
 9341 
 9342  format %{
 9343     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9344     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9345  %}
 9346 
 9347  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9348             aarch64_enc_cset_eq(res));
 9349 
 9350   ins_pipe(pipe_slow);
 9351 %}
 9352 
 9353 
 9354 // ---------------------------------------------------------------------
 9355 
 9356 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9357 
 9358 // Sundry CAS operations.  Note that release is always true,
 9359 // regardless of the memory ordering of the CAS.  This is because we
 9360 // need the volatile case to be sequentially consistent but there is
 9361 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9362 // can't check the type of memory ordering here, so we always emit a
 9363 // STLXR.
 9364 
 9365 // This section is generated from aarch64_ad_cas.m4
 9366 
 9367 
 9368 
 9369 // This pattern is generated automatically from cas.m4.
 9370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9371 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9372 
 9373   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9374   ins_cost(2 * VOLATILE_REF_COST);
 9375   effect(TEMP_DEF res, KILL cr);
 9376   format %{
 9377     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9378   %}
 9379   ins_encode %{
 9380     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9381                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9382                /*weak*/ false, $res$$Register);
 9383     __ sxtbw($res$$Register, $res$$Register);
 9384   %}
 9385   ins_pipe(pipe_slow);
 9386 %}
 9387 
 9388 // This pattern is generated automatically from cas.m4.
 9389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9390 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9391 
 9392   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9393   ins_cost(2 * VOLATILE_REF_COST);
 9394   effect(TEMP_DEF res, KILL cr);
 9395   format %{
 9396     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9397   %}
 9398   ins_encode %{
 9399     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9400                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9401                /*weak*/ false, $res$$Register);
 9402     __ sxthw($res$$Register, $res$$Register);
 9403   %}
 9404   ins_pipe(pipe_slow);
 9405 %}
 9406 
 9407 // This pattern is generated automatically from cas.m4.
 9408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9409 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9410 
 9411   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9412   ins_cost(2 * VOLATILE_REF_COST);
 9413   effect(TEMP_DEF res, KILL cr);
 9414   format %{
 9415     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9416   %}
 9417   ins_encode %{
 9418     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9419                Assembler::word, /*acquire*/ false, /*release*/ true,
 9420                /*weak*/ false, $res$$Register);
 9421   %}
 9422   ins_pipe(pipe_slow);
 9423 %}
 9424 
 9425 // This pattern is generated automatically from cas.m4.
 9426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9427 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9428 
 9429   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9430   ins_cost(2 * VOLATILE_REF_COST);
 9431   effect(TEMP_DEF res, KILL cr);
 9432   format %{
 9433     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9434   %}
 9435   ins_encode %{
 9436     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9437                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9438                /*weak*/ false, $res$$Register);
 9439   %}
 9440   ins_pipe(pipe_slow);
 9441 %}
 9442 
 9443 // This pattern is generated automatically from cas.m4.
 9444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9445 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9446 
 9447   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9448   ins_cost(2 * VOLATILE_REF_COST);
 9449   effect(TEMP_DEF res, KILL cr);
 9450   format %{
 9451     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9452   %}
 9453   ins_encode %{
 9454     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9455                Assembler::word, /*acquire*/ false, /*release*/ true,
 9456                /*weak*/ false, $res$$Register);
 9457   %}
 9458   ins_pipe(pipe_slow);
 9459 %}
 9460 
 9461 // This pattern is generated automatically from cas.m4.
 9462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9463 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9464   predicate(n->as_LoadStore()->barrier_data() == 0);
 9465   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9466   ins_cost(2 * VOLATILE_REF_COST);
 9467   effect(TEMP_DEF res, KILL cr);
 9468   format %{
 9469     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9470   %}
 9471   ins_encode %{
 9472     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9473                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9474                /*weak*/ false, $res$$Register);
 9475   %}
 9476   ins_pipe(pipe_slow);
 9477 %}
 9478 
 9479 // This pattern is generated automatically from cas.m4.
 9480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9481 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9482   predicate(needs_acquiring_load_exclusive(n));
 9483   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9484   ins_cost(VOLATILE_REF_COST);
 9485   effect(TEMP_DEF res, KILL cr);
 9486   format %{
 9487     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9488   %}
 9489   ins_encode %{
 9490     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9491                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9492                /*weak*/ false, $res$$Register);
 9493     __ sxtbw($res$$Register, $res$$Register);
 9494   %}
 9495   ins_pipe(pipe_slow);
 9496 %}
 9497 
 9498 // This pattern is generated automatically from cas.m4.
 9499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9500 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9501   predicate(needs_acquiring_load_exclusive(n));
 9502   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9503   ins_cost(VOLATILE_REF_COST);
 9504   effect(TEMP_DEF res, KILL cr);
 9505   format %{
 9506     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9507   %}
 9508   ins_encode %{
 9509     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9510                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9511                /*weak*/ false, $res$$Register);
 9512     __ sxthw($res$$Register, $res$$Register);
 9513   %}
 9514   ins_pipe(pipe_slow);
 9515 %}
 9516 
 9517 // This pattern is generated automatically from cas.m4.
 9518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9519 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9520   predicate(needs_acquiring_load_exclusive(n));
 9521   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9522   ins_cost(VOLATILE_REF_COST);
 9523   effect(TEMP_DEF res, KILL cr);
 9524   format %{
 9525     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9526   %}
 9527   ins_encode %{
 9528     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9529                Assembler::word, /*acquire*/ true, /*release*/ true,
 9530                /*weak*/ false, $res$$Register);
 9531   %}
 9532   ins_pipe(pipe_slow);
 9533 %}
 9534 
 9535 // This pattern is generated automatically from cas.m4.
 9536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9537 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9538   predicate(needs_acquiring_load_exclusive(n));
 9539   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9540   ins_cost(VOLATILE_REF_COST);
 9541   effect(TEMP_DEF res, KILL cr);
 9542   format %{
 9543     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9544   %}
 9545   ins_encode %{
 9546     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9547                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9548                /*weak*/ false, $res$$Register);
 9549   %}
 9550   ins_pipe(pipe_slow);
 9551 %}
 9552 
 9553 // This pattern is generated automatically from cas.m4.
 9554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9555 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9556   predicate(needs_acquiring_load_exclusive(n));
 9557   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9558   ins_cost(VOLATILE_REF_COST);
 9559   effect(TEMP_DEF res, KILL cr);
 9560   format %{
 9561     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9562   %}
 9563   ins_encode %{
 9564     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9565                Assembler::word, /*acquire*/ true, /*release*/ true,
 9566                /*weak*/ false, $res$$Register);
 9567   %}
 9568   ins_pipe(pipe_slow);
 9569 %}
 9570 
 9571 // This pattern is generated automatically from cas.m4.
 9572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9573 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9574   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9575   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9576   ins_cost(VOLATILE_REF_COST);
 9577   effect(TEMP_DEF res, KILL cr);
 9578   format %{
 9579     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9580   %}
 9581   ins_encode %{
 9582     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9583                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9584                /*weak*/ false, $res$$Register);
 9585   %}
 9586   ins_pipe(pipe_slow);
 9587 %}
 9588 
 9589 // This pattern is generated automatically from cas.m4.
 9590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9591 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9592 
 9593   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9594   ins_cost(2 * VOLATILE_REF_COST);
 9595   effect(KILL cr);
 9596   format %{
 9597     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9598     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9599   %}
 9600   ins_encode %{
 9601     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9602                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9603                /*weak*/ true, noreg);
 9604     __ csetw($res$$Register, Assembler::EQ);
 9605   %}
 9606   ins_pipe(pipe_slow);
 9607 %}
 9608 
 9609 // This pattern is generated automatically from cas.m4.
 9610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9611 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9612 
 9613   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9614   ins_cost(2 * VOLATILE_REF_COST);
 9615   effect(KILL cr);
 9616   format %{
 9617     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9618     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9619   %}
 9620   ins_encode %{
 9621     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9622                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9623                /*weak*/ true, noreg);
 9624     __ csetw($res$$Register, Assembler::EQ);
 9625   %}
 9626   ins_pipe(pipe_slow);
 9627 %}
 9628 
 9629 // This pattern is generated automatically from cas.m4.
 9630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9631 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9632 
 9633   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9634   ins_cost(2 * VOLATILE_REF_COST);
 9635   effect(KILL cr);
 9636   format %{
 9637     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9638     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9639   %}
 9640   ins_encode %{
 9641     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9642                Assembler::word, /*acquire*/ false, /*release*/ true,
 9643                /*weak*/ true, noreg);
 9644     __ csetw($res$$Register, Assembler::EQ);
 9645   %}
 9646   ins_pipe(pipe_slow);
 9647 %}
 9648 
 9649 // This pattern is generated automatically from cas.m4.
 9650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9651 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9652 
 9653   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9654   ins_cost(2 * VOLATILE_REF_COST);
 9655   effect(KILL cr);
 9656   format %{
 9657     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9658     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9659   %}
 9660   ins_encode %{
 9661     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9662                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9663                /*weak*/ true, noreg);
 9664     __ csetw($res$$Register, Assembler::EQ);
 9665   %}
 9666   ins_pipe(pipe_slow);
 9667 %}
 9668 
 9669 // This pattern is generated automatically from cas.m4.
 9670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9671 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9672 
 9673   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9674   ins_cost(2 * VOLATILE_REF_COST);
 9675   effect(KILL cr);
 9676   format %{
 9677     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9678     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9679   %}
 9680   ins_encode %{
 9681     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9682                Assembler::word, /*acquire*/ false, /*release*/ true,
 9683                /*weak*/ true, noreg);
 9684     __ csetw($res$$Register, Assembler::EQ);
 9685   %}
 9686   ins_pipe(pipe_slow);
 9687 %}
 9688 
 9689 // This pattern is generated automatically from cas.m4.
 9690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9691 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9692   predicate(n->as_LoadStore()->barrier_data() == 0);
 9693   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9694   ins_cost(2 * VOLATILE_REF_COST);
 9695   effect(KILL cr);
 9696   format %{
 9697     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9698     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9699   %}
 9700   ins_encode %{
 9701     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9702                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9703                /*weak*/ true, noreg);
 9704     __ csetw($res$$Register, Assembler::EQ);
 9705   %}
 9706   ins_pipe(pipe_slow);
 9707 %}
 9708 
 9709 // This pattern is generated automatically from cas.m4.
 9710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9711 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9712   predicate(needs_acquiring_load_exclusive(n));
 9713   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9714   ins_cost(VOLATILE_REF_COST);
 9715   effect(KILL cr);
 9716   format %{
 9717     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9718     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9719   %}
 9720   ins_encode %{
 9721     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9722                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9723                /*weak*/ true, noreg);
 9724     __ csetw($res$$Register, Assembler::EQ);
 9725   %}
 9726   ins_pipe(pipe_slow);
 9727 %}
 9728 
 9729 // This pattern is generated automatically from cas.m4.
 9730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9731 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9732   predicate(needs_acquiring_load_exclusive(n));
 9733   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9734   ins_cost(VOLATILE_REF_COST);
 9735   effect(KILL cr);
 9736   format %{
 9737     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9738     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9739   %}
 9740   ins_encode %{
 9741     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9742                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9743                /*weak*/ true, noreg);
 9744     __ csetw($res$$Register, Assembler::EQ);
 9745   %}
 9746   ins_pipe(pipe_slow);
 9747 %}
 9748 
 9749 // This pattern is generated automatically from cas.m4.
 9750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9751 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9752   predicate(needs_acquiring_load_exclusive(n));
 9753   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9754   ins_cost(VOLATILE_REF_COST);
 9755   effect(KILL cr);
 9756   format %{
 9757     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9758     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9759   %}
 9760   ins_encode %{
 9761     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9762                Assembler::word, /*acquire*/ true, /*release*/ true,
 9763                /*weak*/ true, noreg);
 9764     __ csetw($res$$Register, Assembler::EQ);
 9765   %}
 9766   ins_pipe(pipe_slow);
 9767 %}
 9768 
 9769 // This pattern is generated automatically from cas.m4.
 9770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9771 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9772   predicate(needs_acquiring_load_exclusive(n));
 9773   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9774   ins_cost(VOLATILE_REF_COST);
 9775   effect(KILL cr);
 9776   format %{
 9777     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9778     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9779   %}
 9780   ins_encode %{
 9781     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9782                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9783                /*weak*/ true, noreg);
 9784     __ csetw($res$$Register, Assembler::EQ);
 9785   %}
 9786   ins_pipe(pipe_slow);
 9787 %}
 9788 
 9789 // This pattern is generated automatically from cas.m4.
 9790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9791 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9792   predicate(needs_acquiring_load_exclusive(n));
 9793   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9794   ins_cost(VOLATILE_REF_COST);
 9795   effect(KILL cr);
 9796   format %{
 9797     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9798     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9799   %}
 9800   ins_encode %{
 9801     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9802                Assembler::word, /*acquire*/ true, /*release*/ true,
 9803                /*weak*/ true, noreg);
 9804     __ csetw($res$$Register, Assembler::EQ);
 9805   %}
 9806   ins_pipe(pipe_slow);
 9807 %}
 9808 
 9809 // This pattern is generated automatically from cas.m4.
 9810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9811 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9812   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9813   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9814   ins_cost(VOLATILE_REF_COST);
 9815   effect(KILL cr);
 9816   format %{
 9817     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9818     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9819   %}
 9820   ins_encode %{
 9821     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9822                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9823                /*weak*/ true, noreg);
 9824     __ csetw($res$$Register, Assembler::EQ);
 9825   %}
 9826   ins_pipe(pipe_slow);
 9827 %}
 9828 
 9829 // END This section of the file is automatically generated. Do not edit --------------
 9830 // ---------------------------------------------------------------------
 9831 
 9832 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9833   match(Set prev (GetAndSetI mem newv));
 9834   ins_cost(2 * VOLATILE_REF_COST);
 9835   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9836   ins_encode %{
 9837     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9838   %}
 9839   ins_pipe(pipe_serial);
 9840 %}
 9841 
 9842 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9843   match(Set prev (GetAndSetL mem newv));
 9844   ins_cost(2 * VOLATILE_REF_COST);
 9845   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9846   ins_encode %{
 9847     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9848   %}
 9849   ins_pipe(pipe_serial);
 9850 %}
 9851 
 9852 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9853   match(Set prev (GetAndSetN mem newv));
 9854   ins_cost(2 * VOLATILE_REF_COST);
 9855   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9856   ins_encode %{
 9857     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9858   %}
 9859   ins_pipe(pipe_serial);
 9860 %}
 9861 
 9862 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9863   predicate(n->as_LoadStore()->barrier_data() == 0);
 9864   match(Set prev (GetAndSetP mem newv));
 9865   ins_cost(2 * VOLATILE_REF_COST);
 9866   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9867   ins_encode %{
 9868     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9869   %}
 9870   ins_pipe(pipe_serial);
 9871 %}
 9872 
 9873 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9874   predicate(needs_acquiring_load_exclusive(n));
 9875   match(Set prev (GetAndSetI mem newv));
 9876   ins_cost(VOLATILE_REF_COST);
 9877   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9878   ins_encode %{
 9879     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9880   %}
 9881   ins_pipe(pipe_serial);
 9882 %}
 9883 
 9884 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9885   predicate(needs_acquiring_load_exclusive(n));
 9886   match(Set prev (GetAndSetL mem newv));
 9887   ins_cost(VOLATILE_REF_COST);
 9888   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9889   ins_encode %{
 9890     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9891   %}
 9892   ins_pipe(pipe_serial);
 9893 %}
 9894 
 9895 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9896   predicate(needs_acquiring_load_exclusive(n));
 9897   match(Set prev (GetAndSetN mem newv));
 9898   ins_cost(VOLATILE_REF_COST);
 9899   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9900   ins_encode %{
 9901     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9902   %}
 9903   ins_pipe(pipe_serial);
 9904 %}
 9905 
 9906 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9907   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9908   match(Set prev (GetAndSetP mem newv));
 9909   ins_cost(VOLATILE_REF_COST);
 9910   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9911   ins_encode %{
 9912     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9913   %}
 9914   ins_pipe(pipe_serial);
 9915 %}
 9916 
 9917 
 9918 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9919   match(Set newval (GetAndAddL mem incr));
 9920   ins_cost(2 * VOLATILE_REF_COST + 1);
 9921   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9922   ins_encode %{
 9923     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9924   %}
 9925   ins_pipe(pipe_serial);
 9926 %}
 9927 
 9928 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9929   predicate(n->as_LoadStore()->result_not_used());
 9930   match(Set dummy (GetAndAddL mem incr));
 9931   ins_cost(2 * VOLATILE_REF_COST);
 9932   format %{ "get_and_addL [$mem], $incr" %}
 9933   ins_encode %{
 9934     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9935   %}
 9936   ins_pipe(pipe_serial);
 9937 %}
 9938 
 9939 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9940   match(Set newval (GetAndAddL mem incr));
 9941   ins_cost(2 * VOLATILE_REF_COST + 1);
 9942   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9943   ins_encode %{
 9944     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9945   %}
 9946   ins_pipe(pipe_serial);
 9947 %}
 9948 
 9949 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9950   predicate(n->as_LoadStore()->result_not_used());
 9951   match(Set dummy (GetAndAddL mem incr));
 9952   ins_cost(2 * VOLATILE_REF_COST);
 9953   format %{ "get_and_addL [$mem], $incr" %}
 9954   ins_encode %{
 9955     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9956   %}
 9957   ins_pipe(pipe_serial);
 9958 %}
 9959 
 9960 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9961   match(Set newval (GetAndAddI mem incr));
 9962   ins_cost(2 * VOLATILE_REF_COST + 1);
 9963   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9964   ins_encode %{
 9965     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9966   %}
 9967   ins_pipe(pipe_serial);
 9968 %}
 9969 
 9970 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9971   predicate(n->as_LoadStore()->result_not_used());
 9972   match(Set dummy (GetAndAddI mem incr));
 9973   ins_cost(2 * VOLATILE_REF_COST);
 9974   format %{ "get_and_addI [$mem], $incr" %}
 9975   ins_encode %{
 9976     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9977   %}
 9978   ins_pipe(pipe_serial);
 9979 %}
 9980 
 9981 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9982   match(Set newval (GetAndAddI mem incr));
 9983   ins_cost(2 * VOLATILE_REF_COST + 1);
 9984   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9985   ins_encode %{
 9986     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9987   %}
 9988   ins_pipe(pipe_serial);
 9989 %}
 9990 
 9991 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9992   predicate(n->as_LoadStore()->result_not_used());
 9993   match(Set dummy (GetAndAddI mem incr));
 9994   ins_cost(2 * VOLATILE_REF_COST);
 9995   format %{ "get_and_addI [$mem], $incr" %}
 9996   ins_encode %{
 9997     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9998   %}
 9999   ins_pipe(pipe_serial);
10000 %}
10001 
10002 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10003   predicate(needs_acquiring_load_exclusive(n));
10004   match(Set newval (GetAndAddL mem incr));
10005   ins_cost(VOLATILE_REF_COST + 1);
10006   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10007   ins_encode %{
10008     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10009   %}
10010   ins_pipe(pipe_serial);
10011 %}
10012 
10013 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10014   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10015   match(Set dummy (GetAndAddL mem incr));
10016   ins_cost(VOLATILE_REF_COST);
10017   format %{ "get_and_addL_acq [$mem], $incr" %}
10018   ins_encode %{
10019     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10020   %}
10021   ins_pipe(pipe_serial);
10022 %}
10023 
10024 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10025   predicate(needs_acquiring_load_exclusive(n));
10026   match(Set newval (GetAndAddL mem incr));
10027   ins_cost(VOLATILE_REF_COST + 1);
10028   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10029   ins_encode %{
10030     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10031   %}
10032   ins_pipe(pipe_serial);
10033 %}
10034 
10035 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10036   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10037   match(Set dummy (GetAndAddL mem incr));
10038   ins_cost(VOLATILE_REF_COST);
10039   format %{ "get_and_addL_acq [$mem], $incr" %}
10040   ins_encode %{
10041     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10042   %}
10043   ins_pipe(pipe_serial);
10044 %}
10045 
10046 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10047   predicate(needs_acquiring_load_exclusive(n));
10048   match(Set newval (GetAndAddI mem incr));
10049   ins_cost(VOLATILE_REF_COST + 1);
10050   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10051   ins_encode %{
10052     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10053   %}
10054   ins_pipe(pipe_serial);
10055 %}
10056 
10057 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10058   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10059   match(Set dummy (GetAndAddI mem incr));
10060   ins_cost(VOLATILE_REF_COST);
10061   format %{ "get_and_addI_acq [$mem], $incr" %}
10062   ins_encode %{
10063     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10064   %}
10065   ins_pipe(pipe_serial);
10066 %}
10067 
10068 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10069   predicate(needs_acquiring_load_exclusive(n));
10070   match(Set newval (GetAndAddI mem incr));
10071   ins_cost(VOLATILE_REF_COST + 1);
10072   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10073   ins_encode %{
10074     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10075   %}
10076   ins_pipe(pipe_serial);
10077 %}
10078 
10079 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10080   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10081   match(Set dummy (GetAndAddI mem incr));
10082   ins_cost(VOLATILE_REF_COST);
10083   format %{ "get_and_addI_acq [$mem], $incr" %}
10084   ins_encode %{
10085     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10086   %}
10087   ins_pipe(pipe_serial);
10088 %}
10089 
10090 // Manifest a CmpL result in an integer register.
10091 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10092 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10093 %{
10094   match(Set dst (CmpL3 src1 src2));
10095   effect(KILL flags);
10096 
10097   ins_cost(INSN_COST * 6);
10098   format %{
10099       "cmp $src1, $src2"
10100       "csetw $dst, ne"
10101       "cnegw $dst, lt"
10102   %}
10103   // format %{ "CmpL3 $dst, $src1, $src2" %}
10104   ins_encode %{
10105     __ cmp($src1$$Register, $src2$$Register);
10106     __ csetw($dst$$Register, Assembler::NE);
10107     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10108   %}
10109 
10110   ins_pipe(pipe_class_default);
10111 %}
10112 
10113 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10114 %{
10115   match(Set dst (CmpL3 src1 src2));
10116   effect(KILL flags);
10117 
10118   ins_cost(INSN_COST * 6);
10119   format %{
10120       "cmp $src1, $src2"
10121       "csetw $dst, ne"
10122       "cnegw $dst, lt"
10123   %}
10124   ins_encode %{
10125     int32_t con = (int32_t)$src2$$constant;
10126      if (con < 0) {
10127       __ adds(zr, $src1$$Register, -con);
10128     } else {
10129       __ subs(zr, $src1$$Register, con);
10130     }
10131     __ csetw($dst$$Register, Assembler::NE);
10132     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10133   %}
10134 
10135   ins_pipe(pipe_class_default);
10136 %}
10137 
10138 // ============================================================================
10139 // Conditional Move Instructions
10140 
10141 // n.b. we have identical rules for both a signed compare op (cmpOp)
10142 // and an unsigned compare op (cmpOpU). it would be nice if we could
10143 // define an op class which merged both inputs and use it to type the
10144 // argument to a single rule. unfortunatelyt his fails because the
10145 // opclass does not live up to the COND_INTER interface of its
10146 // component operands. When the generic code tries to negate the
10147 // operand it ends up running the generci Machoper::negate method
10148 // which throws a ShouldNotHappen. So, we have to provide two flavours
10149 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10150 
10151 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10152   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10153 
10154   ins_cost(INSN_COST * 2);
10155   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10156 
10157   ins_encode %{
10158     __ cselw(as_Register($dst$$reg),
10159              as_Register($src2$$reg),
10160              as_Register($src1$$reg),
10161              (Assembler::Condition)$cmp$$cmpcode);
10162   %}
10163 
10164   ins_pipe(icond_reg_reg);
10165 %}
10166 
10167 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10168   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10169 
10170   ins_cost(INSN_COST * 2);
10171   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10172 
10173   ins_encode %{
10174     __ cselw(as_Register($dst$$reg),
10175              as_Register($src2$$reg),
10176              as_Register($src1$$reg),
10177              (Assembler::Condition)$cmp$$cmpcode);
10178   %}
10179 
10180   ins_pipe(icond_reg_reg);
10181 %}
10182 
10183 // special cases where one arg is zero
10184 
10185 // n.b. this is selected in preference to the rule above because it
10186 // avoids loading constant 0 into a source register
10187 
10188 // TODO
10189 // we ought only to be able to cull one of these variants as the ideal
10190 // transforms ought always to order the zero consistently (to left/right?)
10191 
10192 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10193   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10194 
10195   ins_cost(INSN_COST * 2);
10196   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10197 
10198   ins_encode %{
10199     __ cselw(as_Register($dst$$reg),
10200              as_Register($src$$reg),
10201              zr,
10202              (Assembler::Condition)$cmp$$cmpcode);
10203   %}
10204 
10205   ins_pipe(icond_reg);
10206 %}
10207 
10208 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10209   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10210 
10211   ins_cost(INSN_COST * 2);
10212   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10213 
10214   ins_encode %{
10215     __ cselw(as_Register($dst$$reg),
10216              as_Register($src$$reg),
10217              zr,
10218              (Assembler::Condition)$cmp$$cmpcode);
10219   %}
10220 
10221   ins_pipe(icond_reg);
10222 %}
10223 
10224 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10225   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10226 
10227   ins_cost(INSN_COST * 2);
10228   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10229 
10230   ins_encode %{
10231     __ cselw(as_Register($dst$$reg),
10232              zr,
10233              as_Register($src$$reg),
10234              (Assembler::Condition)$cmp$$cmpcode);
10235   %}
10236 
10237   ins_pipe(icond_reg);
10238 %}
10239 
10240 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10241   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10242 
10243   ins_cost(INSN_COST * 2);
10244   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10245 
10246   ins_encode %{
10247     __ cselw(as_Register($dst$$reg),
10248              zr,
10249              as_Register($src$$reg),
10250              (Assembler::Condition)$cmp$$cmpcode);
10251   %}
10252 
10253   ins_pipe(icond_reg);
10254 %}
10255 
10256 // special case for creating a boolean 0 or 1
10257 
10258 // n.b. this is selected in preference to the rule above because it
10259 // avoids loading constants 0 and 1 into a source register
10260 
10261 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10262   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10263 
10264   ins_cost(INSN_COST * 2);
10265   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10266 
10267   ins_encode %{
10268     // equivalently
10269     // cset(as_Register($dst$$reg),
10270     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10271     __ csincw(as_Register($dst$$reg),
10272              zr,
10273              zr,
10274              (Assembler::Condition)$cmp$$cmpcode);
10275   %}
10276 
10277   ins_pipe(icond_none);
10278 %}
10279 
10280 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10281   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10282 
10283   ins_cost(INSN_COST * 2);
10284   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10285 
10286   ins_encode %{
10287     // equivalently
10288     // cset(as_Register($dst$$reg),
10289     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10290     __ csincw(as_Register($dst$$reg),
10291              zr,
10292              zr,
10293              (Assembler::Condition)$cmp$$cmpcode);
10294   %}
10295 
10296   ins_pipe(icond_none);
10297 %}
10298 
10299 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10300   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10301 
10302   ins_cost(INSN_COST * 2);
10303   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10304 
10305   ins_encode %{
10306     __ csel(as_Register($dst$$reg),
10307             as_Register($src2$$reg),
10308             as_Register($src1$$reg),
10309             (Assembler::Condition)$cmp$$cmpcode);
10310   %}
10311 
10312   ins_pipe(icond_reg_reg);
10313 %}
10314 
10315 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10316   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10317 
10318   ins_cost(INSN_COST * 2);
10319   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10320 
10321   ins_encode %{
10322     __ csel(as_Register($dst$$reg),
10323             as_Register($src2$$reg),
10324             as_Register($src1$$reg),
10325             (Assembler::Condition)$cmp$$cmpcode);
10326   %}
10327 
10328   ins_pipe(icond_reg_reg);
10329 %}
10330 
10331 // special cases where one arg is zero
10332 
10333 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10334   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10335 
10336   ins_cost(INSN_COST * 2);
10337   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10338 
10339   ins_encode %{
10340     __ csel(as_Register($dst$$reg),
10341             zr,
10342             as_Register($src$$reg),
10343             (Assembler::Condition)$cmp$$cmpcode);
10344   %}
10345 
10346   ins_pipe(icond_reg);
10347 %}
10348 
10349 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10350   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10351 
10352   ins_cost(INSN_COST * 2);
10353   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10354 
10355   ins_encode %{
10356     __ csel(as_Register($dst$$reg),
10357             zr,
10358             as_Register($src$$reg),
10359             (Assembler::Condition)$cmp$$cmpcode);
10360   %}
10361 
10362   ins_pipe(icond_reg);
10363 %}
10364 
10365 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10366   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10367 
10368   ins_cost(INSN_COST * 2);
10369   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10370 
10371   ins_encode %{
10372     __ csel(as_Register($dst$$reg),
10373             as_Register($src$$reg),
10374             zr,
10375             (Assembler::Condition)$cmp$$cmpcode);
10376   %}
10377 
10378   ins_pipe(icond_reg);
10379 %}
10380 
10381 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10382   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10383 
10384   ins_cost(INSN_COST * 2);
10385   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10386 
10387   ins_encode %{
10388     __ csel(as_Register($dst$$reg),
10389             as_Register($src$$reg),
10390             zr,
10391             (Assembler::Condition)$cmp$$cmpcode);
10392   %}
10393 
10394   ins_pipe(icond_reg);
10395 %}
10396 
10397 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10398   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10399 
10400   ins_cost(INSN_COST * 2);
10401   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10402 
10403   ins_encode %{
10404     __ csel(as_Register($dst$$reg),
10405             as_Register($src2$$reg),
10406             as_Register($src1$$reg),
10407             (Assembler::Condition)$cmp$$cmpcode);
10408   %}
10409 
10410   ins_pipe(icond_reg_reg);
10411 %}
10412 
10413 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10414   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10415 
10416   ins_cost(INSN_COST * 2);
10417   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10418 
10419   ins_encode %{
10420     __ csel(as_Register($dst$$reg),
10421             as_Register($src2$$reg),
10422             as_Register($src1$$reg),
10423             (Assembler::Condition)$cmp$$cmpcode);
10424   %}
10425 
10426   ins_pipe(icond_reg_reg);
10427 %}
10428 
10429 // special cases where one arg is zero
10430 
10431 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10432   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10433 
10434   ins_cost(INSN_COST * 2);
10435   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10436 
10437   ins_encode %{
10438     __ csel(as_Register($dst$$reg),
10439             zr,
10440             as_Register($src$$reg),
10441             (Assembler::Condition)$cmp$$cmpcode);
10442   %}
10443 
10444   ins_pipe(icond_reg);
10445 %}
10446 
10447 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10448   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10449 
10450   ins_cost(INSN_COST * 2);
10451   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10452 
10453   ins_encode %{
10454     __ csel(as_Register($dst$$reg),
10455             zr,
10456             as_Register($src$$reg),
10457             (Assembler::Condition)$cmp$$cmpcode);
10458   %}
10459 
10460   ins_pipe(icond_reg);
10461 %}
10462 
10463 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10464   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10465 
10466   ins_cost(INSN_COST * 2);
10467   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10468 
10469   ins_encode %{
10470     __ csel(as_Register($dst$$reg),
10471             as_Register($src$$reg),
10472             zr,
10473             (Assembler::Condition)$cmp$$cmpcode);
10474   %}
10475 
10476   ins_pipe(icond_reg);
10477 %}
10478 
10479 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10480   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10481 
10482   ins_cost(INSN_COST * 2);
10483   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10484 
10485   ins_encode %{
10486     __ csel(as_Register($dst$$reg),
10487             as_Register($src$$reg),
10488             zr,
10489             (Assembler::Condition)$cmp$$cmpcode);
10490   %}
10491 
10492   ins_pipe(icond_reg);
10493 %}
10494 
10495 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10496   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10497 
10498   ins_cost(INSN_COST * 2);
10499   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10500 
10501   ins_encode %{
10502     __ cselw(as_Register($dst$$reg),
10503              as_Register($src2$$reg),
10504              as_Register($src1$$reg),
10505              (Assembler::Condition)$cmp$$cmpcode);
10506   %}
10507 
10508   ins_pipe(icond_reg_reg);
10509 %}
10510 
10511 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10512   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10513 
10514   ins_cost(INSN_COST * 2);
10515   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10516 
10517   ins_encode %{
10518     __ cselw(as_Register($dst$$reg),
10519              as_Register($src2$$reg),
10520              as_Register($src1$$reg),
10521              (Assembler::Condition)$cmp$$cmpcode);
10522   %}
10523 
10524   ins_pipe(icond_reg_reg);
10525 %}
10526 
10527 // special cases where one arg is zero
10528 
10529 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10530   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10531 
10532   ins_cost(INSN_COST * 2);
10533   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10534 
10535   ins_encode %{
10536     __ cselw(as_Register($dst$$reg),
10537              zr,
10538              as_Register($src$$reg),
10539              (Assembler::Condition)$cmp$$cmpcode);
10540   %}
10541 
10542   ins_pipe(icond_reg);
10543 %}
10544 
10545 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10546   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10547 
10548   ins_cost(INSN_COST * 2);
10549   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10550 
10551   ins_encode %{
10552     __ cselw(as_Register($dst$$reg),
10553              zr,
10554              as_Register($src$$reg),
10555              (Assembler::Condition)$cmp$$cmpcode);
10556   %}
10557 
10558   ins_pipe(icond_reg);
10559 %}
10560 
10561 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10562   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10563 
10564   ins_cost(INSN_COST * 2);
10565   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10566 
10567   ins_encode %{
10568     __ cselw(as_Register($dst$$reg),
10569              as_Register($src$$reg),
10570              zr,
10571              (Assembler::Condition)$cmp$$cmpcode);
10572   %}
10573 
10574   ins_pipe(icond_reg);
10575 %}
10576 
10577 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10578   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10579 
10580   ins_cost(INSN_COST * 2);
10581   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10582 
10583   ins_encode %{
10584     __ cselw(as_Register($dst$$reg),
10585              as_Register($src$$reg),
10586              zr,
10587              (Assembler::Condition)$cmp$$cmpcode);
10588   %}
10589 
10590   ins_pipe(icond_reg);
10591 %}
10592 
10593 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10594 %{
10595   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10596 
10597   ins_cost(INSN_COST * 3);
10598 
10599   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10600   ins_encode %{
10601     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10602     __ fcsels(as_FloatRegister($dst$$reg),
10603               as_FloatRegister($src2$$reg),
10604               as_FloatRegister($src1$$reg),
10605               cond);
10606   %}
10607 
10608   ins_pipe(fp_cond_reg_reg_s);
10609 %}
10610 
10611 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10612 %{
10613   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10614 
10615   ins_cost(INSN_COST * 3);
10616 
10617   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10618   ins_encode %{
10619     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10620     __ fcsels(as_FloatRegister($dst$$reg),
10621               as_FloatRegister($src2$$reg),
10622               as_FloatRegister($src1$$reg),
10623               cond);
10624   %}
10625 
10626   ins_pipe(fp_cond_reg_reg_s);
10627 %}
10628 
10629 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10630 %{
10631   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10632 
10633   ins_cost(INSN_COST * 3);
10634 
10635   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10636   ins_encode %{
10637     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10638     __ fcseld(as_FloatRegister($dst$$reg),
10639               as_FloatRegister($src2$$reg),
10640               as_FloatRegister($src1$$reg),
10641               cond);
10642   %}
10643 
10644   ins_pipe(fp_cond_reg_reg_d);
10645 %}
10646 
10647 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10648 %{
10649   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10650 
10651   ins_cost(INSN_COST * 3);
10652 
10653   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10654   ins_encode %{
10655     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10656     __ fcseld(as_FloatRegister($dst$$reg),
10657               as_FloatRegister($src2$$reg),
10658               as_FloatRegister($src1$$reg),
10659               cond);
10660   %}
10661 
10662   ins_pipe(fp_cond_reg_reg_d);
10663 %}
10664 
10665 // ============================================================================
10666 // Arithmetic Instructions
10667 //
10668 
10669 // Integer Addition
10670 
10671 // TODO
10672 // these currently employ operations which do not set CR and hence are
10673 // not flagged as killing CR but we would like to isolate the cases
10674 // where we want to set flags from those where we don't. need to work
10675 // out how to do that.
10676 
10677 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10678   match(Set dst (AddI src1 src2));
10679 
10680   ins_cost(INSN_COST);
10681   format %{ "addw  $dst, $src1, $src2" %}
10682 
10683   ins_encode %{
10684     __ addw(as_Register($dst$$reg),
10685             as_Register($src1$$reg),
10686             as_Register($src2$$reg));
10687   %}
10688 
10689   ins_pipe(ialu_reg_reg);
10690 %}
10691 
10692 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10693   match(Set dst (AddI src1 src2));
10694 
10695   ins_cost(INSN_COST);
10696   format %{ "addw $dst, $src1, $src2" %}
10697 
10698   // use opcode to indicate that this is an add not a sub
10699   opcode(0x0);
10700 
10701   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10702 
10703   ins_pipe(ialu_reg_imm);
10704 %}
10705 
10706 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10707   match(Set dst (AddI (ConvL2I src1) src2));
10708 
10709   ins_cost(INSN_COST);
10710   format %{ "addw $dst, $src1, $src2" %}
10711 
10712   // use opcode to indicate that this is an add not a sub
10713   opcode(0x0);
10714 
10715   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10716 
10717   ins_pipe(ialu_reg_imm);
10718 %}
10719 
10720 // Pointer Addition
10721 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10722   match(Set dst (AddP src1 src2));
10723 
10724   ins_cost(INSN_COST);
10725   format %{ "add $dst, $src1, $src2\t# ptr" %}
10726 
10727   ins_encode %{
10728     __ add(as_Register($dst$$reg),
10729            as_Register($src1$$reg),
10730            as_Register($src2$$reg));
10731   %}
10732 
10733   ins_pipe(ialu_reg_reg);
10734 %}
10735 
10736 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10737   match(Set dst (AddP src1 (ConvI2L src2)));
10738 
10739   ins_cost(1.9 * INSN_COST);
10740   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10741 
10742   ins_encode %{
10743     __ add(as_Register($dst$$reg),
10744            as_Register($src1$$reg),
10745            as_Register($src2$$reg), ext::sxtw);
10746   %}
10747 
10748   ins_pipe(ialu_reg_reg);
10749 %}
10750 
10751 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10752   match(Set dst (AddP src1 (LShiftL src2 scale)));
10753 
10754   ins_cost(1.9 * INSN_COST);
10755   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10756 
10757   ins_encode %{
10758     __ lea(as_Register($dst$$reg),
10759            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10760                    Address::lsl($scale$$constant)));
10761   %}
10762 
10763   ins_pipe(ialu_reg_reg_shift);
10764 %}
10765 
10766 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10767   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10768 
10769   ins_cost(1.9 * INSN_COST);
10770   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10771 
10772   ins_encode %{
10773     __ lea(as_Register($dst$$reg),
10774            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10775                    Address::sxtw($scale$$constant)));
10776   %}
10777 
10778   ins_pipe(ialu_reg_reg_shift);
10779 %}
10780 
10781 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10782   match(Set dst (LShiftL (ConvI2L src) scale));
10783 
10784   ins_cost(INSN_COST);
10785   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10786 
10787   ins_encode %{
10788     __ sbfiz(as_Register($dst$$reg),
10789           as_Register($src$$reg),
10790           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10791   %}
10792 
10793   ins_pipe(ialu_reg_shift);
10794 %}
10795 
10796 // Pointer Immediate Addition
10797 // n.b. this needs to be more expensive than using an indirect memory
10798 // operand
10799 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10800   match(Set dst (AddP src1 src2));
10801 
10802   ins_cost(INSN_COST);
10803   format %{ "add $dst, $src1, $src2\t# ptr" %}
10804 
10805   // use opcode to indicate that this is an add not a sub
10806   opcode(0x0);
10807 
10808   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10809 
10810   ins_pipe(ialu_reg_imm);
10811 %}
10812 
10813 // Long Addition
10814 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10815 
10816   match(Set dst (AddL src1 src2));
10817 
10818   ins_cost(INSN_COST);
10819   format %{ "add  $dst, $src1, $src2" %}
10820 
10821   ins_encode %{
10822     __ add(as_Register($dst$$reg),
10823            as_Register($src1$$reg),
10824            as_Register($src2$$reg));
10825   %}
10826 
10827   ins_pipe(ialu_reg_reg);
10828 %}
10829 
10830 // No constant pool entries requiredLong Immediate Addition.
10831 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10832   match(Set dst (AddL src1 src2));
10833 
10834   ins_cost(INSN_COST);
10835   format %{ "add $dst, $src1, $src2" %}
10836 
10837   // use opcode to indicate that this is an add not a sub
10838   opcode(0x0);
10839 
10840   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10841 
10842   ins_pipe(ialu_reg_imm);
10843 %}
10844 
10845 // Integer Subtraction
10846 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10847   match(Set dst (SubI src1 src2));
10848 
10849   ins_cost(INSN_COST);
10850   format %{ "subw  $dst, $src1, $src2" %}
10851 
10852   ins_encode %{
10853     __ subw(as_Register($dst$$reg),
10854             as_Register($src1$$reg),
10855             as_Register($src2$$reg));
10856   %}
10857 
10858   ins_pipe(ialu_reg_reg);
10859 %}
10860 
10861 // Immediate Subtraction
10862 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10863   match(Set dst (SubI src1 src2));
10864 
10865   ins_cost(INSN_COST);
10866   format %{ "subw $dst, $src1, $src2" %}
10867 
10868   // use opcode to indicate that this is a sub not an add
10869   opcode(0x1);
10870 
10871   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10872 
10873   ins_pipe(ialu_reg_imm);
10874 %}
10875 
10876 // Long Subtraction
10877 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10878 
10879   match(Set dst (SubL src1 src2));
10880 
10881   ins_cost(INSN_COST);
10882   format %{ "sub  $dst, $src1, $src2" %}
10883 
10884   ins_encode %{
10885     __ sub(as_Register($dst$$reg),
10886            as_Register($src1$$reg),
10887            as_Register($src2$$reg));
10888   %}
10889 
10890   ins_pipe(ialu_reg_reg);
10891 %}
10892 
10893 // No constant pool entries requiredLong Immediate Subtraction.
10894 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10895   match(Set dst (SubL src1 src2));
10896 
10897   ins_cost(INSN_COST);
10898   format %{ "sub$dst, $src1, $src2" %}
10899 
10900   // use opcode to indicate that this is a sub not an add
10901   opcode(0x1);
10902 
10903   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10904 
10905   ins_pipe(ialu_reg_imm);
10906 %}
10907 
10908 // Integer Negation (special case for sub)
10909 
10910 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10911   match(Set dst (SubI zero src));
10912 
10913   ins_cost(INSN_COST);
10914   format %{ "negw $dst, $src\t# int" %}
10915 
10916   ins_encode %{
10917     __ negw(as_Register($dst$$reg),
10918             as_Register($src$$reg));
10919   %}
10920 
10921   ins_pipe(ialu_reg);
10922 %}
10923 
10924 // Long Negation
10925 
10926 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10927   match(Set dst (SubL zero src));
10928 
10929   ins_cost(INSN_COST);
10930   format %{ "neg $dst, $src\t# long" %}
10931 
10932   ins_encode %{
10933     __ neg(as_Register($dst$$reg),
10934            as_Register($src$$reg));
10935   %}
10936 
10937   ins_pipe(ialu_reg);
10938 %}
10939 
10940 // Integer Multiply
10941 
10942 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10943   match(Set dst (MulI src1 src2));
10944 
10945   ins_cost(INSN_COST * 3);
10946   format %{ "mulw  $dst, $src1, $src2" %}
10947 
10948   ins_encode %{
10949     __ mulw(as_Register($dst$$reg),
10950             as_Register($src1$$reg),
10951             as_Register($src2$$reg));
10952   %}
10953 
10954   ins_pipe(imul_reg_reg);
10955 %}
10956 
10957 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10958   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10959 
10960   ins_cost(INSN_COST * 3);
10961   format %{ "smull  $dst, $src1, $src2" %}
10962 
10963   ins_encode %{
10964     __ smull(as_Register($dst$$reg),
10965              as_Register($src1$$reg),
10966              as_Register($src2$$reg));
10967   %}
10968 
10969   ins_pipe(imul_reg_reg);
10970 %}
10971 
10972 // Long Multiply
10973 
10974 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10975   match(Set dst (MulL src1 src2));
10976 
10977   ins_cost(INSN_COST * 5);
10978   format %{ "mul  $dst, $src1, $src2" %}
10979 
10980   ins_encode %{
10981     __ mul(as_Register($dst$$reg),
10982            as_Register($src1$$reg),
10983            as_Register($src2$$reg));
10984   %}
10985 
10986   ins_pipe(lmul_reg_reg);
10987 %}
10988 
10989 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10990 %{
10991   match(Set dst (MulHiL src1 src2));
10992 
10993   ins_cost(INSN_COST * 7);
10994   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10995 
10996   ins_encode %{
10997     __ smulh(as_Register($dst$$reg),
10998              as_Register($src1$$reg),
10999              as_Register($src2$$reg));
11000   %}
11001 
11002   ins_pipe(lmul_reg_reg);
11003 %}
11004 
11005 // Combined Integer Multiply & Add/Sub
11006 
11007 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11008   match(Set dst (AddI src3 (MulI src1 src2)));
11009 
11010   ins_cost(INSN_COST * 3);
11011   format %{ "madd  $dst, $src1, $src2, $src3" %}
11012 
11013   ins_encode %{
11014     __ maddw(as_Register($dst$$reg),
11015              as_Register($src1$$reg),
11016              as_Register($src2$$reg),
11017              as_Register($src3$$reg));
11018   %}
11019 
11020   ins_pipe(imac_reg_reg);
11021 %}
11022 
11023 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11024   match(Set dst (SubI src3 (MulI src1 src2)));
11025 
11026   ins_cost(INSN_COST * 3);
11027   format %{ "msub  $dst, $src1, $src2, $src3" %}
11028 
11029   ins_encode %{
11030     __ msubw(as_Register($dst$$reg),
11031              as_Register($src1$$reg),
11032              as_Register($src2$$reg),
11033              as_Register($src3$$reg));
11034   %}
11035 
11036   ins_pipe(imac_reg_reg);
11037 %}
11038 
11039 // Combined Integer Multiply & Neg
11040 
11041 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11042   match(Set dst (MulI (SubI zero src1) src2));
11043   match(Set dst (MulI src1 (SubI zero src2)));
11044 
11045   ins_cost(INSN_COST * 3);
11046   format %{ "mneg  $dst, $src1, $src2" %}
11047 
11048   ins_encode %{
11049     __ mnegw(as_Register($dst$$reg),
11050              as_Register($src1$$reg),
11051              as_Register($src2$$reg));
11052   %}
11053 
11054   ins_pipe(imac_reg_reg);
11055 %}
11056 
11057 // Combined Long Multiply & Add/Sub
11058 
11059 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11060   match(Set dst (AddL src3 (MulL src1 src2)));
11061 
11062   ins_cost(INSN_COST * 5);
11063   format %{ "madd  $dst, $src1, $src2, $src3" %}
11064 
11065   ins_encode %{
11066     __ madd(as_Register($dst$$reg),
11067             as_Register($src1$$reg),
11068             as_Register($src2$$reg),
11069             as_Register($src3$$reg));
11070   %}
11071 
11072   ins_pipe(lmac_reg_reg);
11073 %}
11074 
11075 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11076   match(Set dst (SubL src3 (MulL src1 src2)));
11077 
11078   ins_cost(INSN_COST * 5);
11079   format %{ "msub  $dst, $src1, $src2, $src3" %}
11080 
11081   ins_encode %{
11082     __ msub(as_Register($dst$$reg),
11083             as_Register($src1$$reg),
11084             as_Register($src2$$reg),
11085             as_Register($src3$$reg));
11086   %}
11087 
11088   ins_pipe(lmac_reg_reg);
11089 %}
11090 
11091 // Combined Long Multiply & Neg
11092 
11093 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11094   match(Set dst (MulL (SubL zero src1) src2));
11095   match(Set dst (MulL src1 (SubL zero src2)));
11096 
11097   ins_cost(INSN_COST * 5);
11098   format %{ "mneg  $dst, $src1, $src2" %}
11099 
11100   ins_encode %{
11101     __ mneg(as_Register($dst$$reg),
11102             as_Register($src1$$reg),
11103             as_Register($src2$$reg));
11104   %}
11105 
11106   ins_pipe(lmac_reg_reg);
11107 %}
11108 
11109 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11110 
11111 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11112   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11113 
11114   ins_cost(INSN_COST * 3);
11115   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11116 
11117   ins_encode %{
11118     __ smaddl(as_Register($dst$$reg),
11119               as_Register($src1$$reg),
11120               as_Register($src2$$reg),
11121               as_Register($src3$$reg));
11122   %}
11123 
11124   ins_pipe(imac_reg_reg);
11125 %}
11126 
11127 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11128   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11129 
11130   ins_cost(INSN_COST * 3);
11131   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11132 
11133   ins_encode %{
11134     __ smsubl(as_Register($dst$$reg),
11135               as_Register($src1$$reg),
11136               as_Register($src2$$reg),
11137               as_Register($src3$$reg));
11138   %}
11139 
11140   ins_pipe(imac_reg_reg);
11141 %}
11142 
11143 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11144   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11145   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11146 
11147   ins_cost(INSN_COST * 3);
11148   format %{ "smnegl  $dst, $src1, $src2" %}
11149 
11150   ins_encode %{
11151     __ smnegl(as_Register($dst$$reg),
11152               as_Register($src1$$reg),
11153               as_Register($src2$$reg));
11154   %}
11155 
11156   ins_pipe(imac_reg_reg);
11157 %}
11158 
11159 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11160 
11161 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11162   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11163 
11164   ins_cost(INSN_COST * 5);
11165   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11166             "maddw $dst, $src3, $src4, rscratch1" %}
11167 
11168   ins_encode %{
11169     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11170     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11171 
11172   ins_pipe(imac_reg_reg);
11173 %}
11174 
11175 // Integer Divide
11176 
11177 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11178   match(Set dst (DivI src1 src2));
11179 
11180   ins_cost(INSN_COST * 19);
11181   format %{ "sdivw  $dst, $src1, $src2" %}
11182 
11183   ins_encode(aarch64_enc_divw(dst, src1, src2));
11184   ins_pipe(idiv_reg_reg);
11185 %}
11186 
11187 // Long Divide
11188 
11189 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11190   match(Set dst (DivL src1 src2));
11191 
11192   ins_cost(INSN_COST * 35);
11193   format %{ "sdiv   $dst, $src1, $src2" %}
11194 
11195   ins_encode(aarch64_enc_div(dst, src1, src2));
11196   ins_pipe(ldiv_reg_reg);
11197 %}
11198 
11199 // Integer Remainder
11200 
11201 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11202   match(Set dst (ModI src1 src2));
11203 
11204   ins_cost(INSN_COST * 22);
11205   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11206             "msubw($dst, rscratch1, $src2, $src1" %}
11207 
11208   ins_encode(aarch64_enc_modw(dst, src1, src2));
11209   ins_pipe(idiv_reg_reg);
11210 %}
11211 
11212 // Long Remainder
11213 
11214 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11215   match(Set dst (ModL src1 src2));
11216 
11217   ins_cost(INSN_COST * 38);
11218   format %{ "sdiv   rscratch1, $src1, $src2\n"
11219             "msub($dst, rscratch1, $src2, $src1" %}
11220 
11221   ins_encode(aarch64_enc_mod(dst, src1, src2));
11222   ins_pipe(ldiv_reg_reg);
11223 %}
11224 
11225 // Integer Shifts
11226 
11227 // Shift Left Register
11228 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11229   match(Set dst (LShiftI src1 src2));
11230 
11231   ins_cost(INSN_COST * 2);
11232   format %{ "lslvw  $dst, $src1, $src2" %}
11233 
11234   ins_encode %{
11235     __ lslvw(as_Register($dst$$reg),
11236              as_Register($src1$$reg),
11237              as_Register($src2$$reg));
11238   %}
11239 
11240   ins_pipe(ialu_reg_reg_vshift);
11241 %}
11242 
11243 // Shift Left Immediate
11244 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11245   match(Set dst (LShiftI src1 src2));
11246 
11247   ins_cost(INSN_COST);
11248   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11249 
11250   ins_encode %{
11251     __ lslw(as_Register($dst$$reg),
11252             as_Register($src1$$reg),
11253             $src2$$constant & 0x1f);
11254   %}
11255 
11256   ins_pipe(ialu_reg_shift);
11257 %}
11258 
11259 // Shift Right Logical Register
11260 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11261   match(Set dst (URShiftI src1 src2));
11262 
11263   ins_cost(INSN_COST * 2);
11264   format %{ "lsrvw  $dst, $src1, $src2" %}
11265 
11266   ins_encode %{
11267     __ lsrvw(as_Register($dst$$reg),
11268              as_Register($src1$$reg),
11269              as_Register($src2$$reg));
11270   %}
11271 
11272   ins_pipe(ialu_reg_reg_vshift);
11273 %}
11274 
11275 // Shift Right Logical Immediate
11276 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11277   match(Set dst (URShiftI src1 src2));
11278 
11279   ins_cost(INSN_COST);
11280   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11281 
11282   ins_encode %{
11283     __ lsrw(as_Register($dst$$reg),
11284             as_Register($src1$$reg),
11285             $src2$$constant & 0x1f);
11286   %}
11287 
11288   ins_pipe(ialu_reg_shift);
11289 %}
11290 
11291 // Shift Right Arithmetic Register
11292 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11293   match(Set dst (RShiftI src1 src2));
11294 
11295   ins_cost(INSN_COST * 2);
11296   format %{ "asrvw  $dst, $src1, $src2" %}
11297 
11298   ins_encode %{
11299     __ asrvw(as_Register($dst$$reg),
11300              as_Register($src1$$reg),
11301              as_Register($src2$$reg));
11302   %}
11303 
11304   ins_pipe(ialu_reg_reg_vshift);
11305 %}
11306 
11307 // Shift Right Arithmetic Immediate
11308 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11309   match(Set dst (RShiftI src1 src2));
11310 
11311   ins_cost(INSN_COST);
11312   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11313 
11314   ins_encode %{
11315     __ asrw(as_Register($dst$$reg),
11316             as_Register($src1$$reg),
11317             $src2$$constant & 0x1f);
11318   %}
11319 
11320   ins_pipe(ialu_reg_shift);
11321 %}
11322 
11323 // Combined Int Mask and Right Shift (using UBFM)
11324 // TODO
11325 
11326 // Long Shifts
11327 
11328 // Shift Left Register
11329 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11330   match(Set dst (LShiftL src1 src2));
11331 
11332   ins_cost(INSN_COST * 2);
11333   format %{ "lslv  $dst, $src1, $src2" %}
11334 
11335   ins_encode %{
11336     __ lslv(as_Register($dst$$reg),
11337             as_Register($src1$$reg),
11338             as_Register($src2$$reg));
11339   %}
11340 
11341   ins_pipe(ialu_reg_reg_vshift);
11342 %}
11343 
11344 // Shift Left Immediate
11345 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11346   match(Set dst (LShiftL src1 src2));
11347 
11348   ins_cost(INSN_COST);
11349   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11350 
11351   ins_encode %{
11352     __ lsl(as_Register($dst$$reg),
11353             as_Register($src1$$reg),
11354             $src2$$constant & 0x3f);
11355   %}
11356 
11357   ins_pipe(ialu_reg_shift);
11358 %}
11359 
11360 // Shift Right Logical Register
11361 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11362   match(Set dst (URShiftL src1 src2));
11363 
11364   ins_cost(INSN_COST * 2);
11365   format %{ "lsrv  $dst, $src1, $src2" %}
11366 
11367   ins_encode %{
11368     __ lsrv(as_Register($dst$$reg),
11369             as_Register($src1$$reg),
11370             as_Register($src2$$reg));
11371   %}
11372 
11373   ins_pipe(ialu_reg_reg_vshift);
11374 %}
11375 
11376 // Shift Right Logical Immediate
11377 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11378   match(Set dst (URShiftL src1 src2));
11379 
11380   ins_cost(INSN_COST);
11381   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11382 
11383   ins_encode %{
11384     __ lsr(as_Register($dst$$reg),
11385            as_Register($src1$$reg),
11386            $src2$$constant & 0x3f);
11387   %}
11388 
11389   ins_pipe(ialu_reg_shift);
11390 %}
11391 
11392 // A special-case pattern for card table stores.
11393 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11394   match(Set dst (URShiftL (CastP2X src1) src2));
11395 
11396   ins_cost(INSN_COST);
11397   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11398 
11399   ins_encode %{
11400     __ lsr(as_Register($dst$$reg),
11401            as_Register($src1$$reg),
11402            $src2$$constant & 0x3f);
11403   %}
11404 
11405   ins_pipe(ialu_reg_shift);
11406 %}
11407 
11408 // Shift Right Arithmetic Register
11409 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11410   match(Set dst (RShiftL src1 src2));
11411 
11412   ins_cost(INSN_COST * 2);
11413   format %{ "asrv  $dst, $src1, $src2" %}
11414 
11415   ins_encode %{
11416     __ asrv(as_Register($dst$$reg),
11417             as_Register($src1$$reg),
11418             as_Register($src2$$reg));
11419   %}
11420 
11421   ins_pipe(ialu_reg_reg_vshift);
11422 %}
11423 
11424 // Shift Right Arithmetic Immediate
11425 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11426   match(Set dst (RShiftL src1 src2));
11427 
11428   ins_cost(INSN_COST);
11429   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11430 
11431   ins_encode %{
11432     __ asr(as_Register($dst$$reg),
11433            as_Register($src1$$reg),
11434            $src2$$constant & 0x3f);
11435   %}
11436 
11437   ins_pipe(ialu_reg_shift);
11438 %}
11439 
11440 // BEGIN This section of the file is automatically generated. Do not edit --------------
11441 // This section is generated from aarch64_ad.m4
11442 
11443 
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446 instruct regL_not_reg(iRegLNoSp dst,
11447                          iRegL src1, immL_M1 m1,
11448                          rFlagsReg cr) %{
11449   match(Set dst (XorL src1 m1));
11450   ins_cost(INSN_COST);
11451   format %{ "eon  $dst, $src1, zr" %}
11452 
11453   ins_encode %{
11454     __ eon(as_Register($dst$$reg),
11455               as_Register($src1$$reg),
11456               zr,
11457               Assembler::LSL, 0);
11458   %}
11459 
11460   ins_pipe(ialu_reg);
11461 %}
11462 
11463 // This pattern is automatically generated from aarch64_ad.m4
11464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11465 instruct regI_not_reg(iRegINoSp dst,
11466                          iRegIorL2I src1, immI_M1 m1,
11467                          rFlagsReg cr) %{
11468   match(Set dst (XorI src1 m1));
11469   ins_cost(INSN_COST);
11470   format %{ "eonw  $dst, $src1, zr" %}
11471 
11472   ins_encode %{
11473     __ eonw(as_Register($dst$$reg),
11474               as_Register($src1$$reg),
11475               zr,
11476               Assembler::LSL, 0);
11477   %}
11478 
11479   ins_pipe(ialu_reg);
11480 %}
11481 
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11485                               immI0 zero, iRegIorL2I src1, immI src2) %{
11486   match(Set dst (SubI zero (URShiftI src1 src2)));
11487 
11488   ins_cost(1.9 * INSN_COST);
11489   format %{ "negw  $dst, $src1, LSR $src2" %}
11490 
11491   ins_encode %{
11492     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11493             Assembler::LSR, $src2$$constant & 0x1f);
11494   %}
11495 
11496   ins_pipe(ialu_reg_shift);
11497 %}
11498 
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11502                               immI0 zero, iRegIorL2I src1, immI src2) %{
11503   match(Set dst (SubI zero (RShiftI src1 src2)));
11504 
11505   ins_cost(1.9 * INSN_COST);
11506   format %{ "negw  $dst, $src1, ASR $src2" %}
11507 
11508   ins_encode %{
11509     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11510             Assembler::ASR, $src2$$constant & 0x1f);
11511   %}
11512 
11513   ins_pipe(ialu_reg_shift);
11514 %}
11515 
11516 // This pattern is automatically generated from aarch64_ad.m4
11517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11518 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11519                               immI0 zero, iRegIorL2I src1, immI src2) %{
11520   match(Set dst (SubI zero (LShiftI src1 src2)));
11521 
11522   ins_cost(1.9 * INSN_COST);
11523   format %{ "negw  $dst, $src1, LSL $src2" %}
11524 
11525   ins_encode %{
11526     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11527             Assembler::LSL, $src2$$constant & 0x1f);
11528   %}
11529 
11530   ins_pipe(ialu_reg_shift);
11531 %}
11532 
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11536                               immL0 zero, iRegL src1, immI src2) %{
11537   match(Set dst (SubL zero (URShiftL src1 src2)));
11538 
11539   ins_cost(1.9 * INSN_COST);
11540   format %{ "neg  $dst, $src1, LSR $src2" %}
11541 
11542   ins_encode %{
11543     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11544             Assembler::LSR, $src2$$constant & 0x3f);
11545   %}
11546 
11547   ins_pipe(ialu_reg_shift);
11548 %}
11549 
11550 // This pattern is automatically generated from aarch64_ad.m4
11551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11552 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11553                               immL0 zero, iRegL src1, immI src2) %{
11554   match(Set dst (SubL zero (RShiftL src1 src2)));
11555 
11556   ins_cost(1.9 * INSN_COST);
11557   format %{ "neg  $dst, $src1, ASR $src2" %}
11558 
11559   ins_encode %{
11560     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11561             Assembler::ASR, $src2$$constant & 0x3f);
11562   %}
11563 
11564   ins_pipe(ialu_reg_shift);
11565 %}
11566 
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11570                               immL0 zero, iRegL src1, immI src2) %{
11571   match(Set dst (SubL zero (LShiftL src1 src2)));
11572 
11573   ins_cost(1.9 * INSN_COST);
11574   format %{ "neg  $dst, $src1, LSL $src2" %}
11575 
11576   ins_encode %{
11577     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11578             Assembler::LSL, $src2$$constant & 0x3f);
11579   %}
11580 
11581   ins_pipe(ialu_reg_shift);
11582 %}
11583 
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct AndI_reg_not_reg(iRegINoSp dst,
11587                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11588   match(Set dst (AndI src1 (XorI src2 m1)));
11589   ins_cost(INSN_COST);
11590   format %{ "bicw  $dst, $src1, $src2" %}
11591 
11592   ins_encode %{
11593     __ bicw(as_Register($dst$$reg),
11594               as_Register($src1$$reg),
11595               as_Register($src2$$reg),
11596               Assembler::LSL, 0);
11597   %}
11598 
11599   ins_pipe(ialu_reg_reg);
11600 %}
11601 
11602 // This pattern is automatically generated from aarch64_ad.m4
11603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11604 instruct AndL_reg_not_reg(iRegLNoSp dst,
11605                          iRegL src1, iRegL src2, immL_M1 m1) %{
11606   match(Set dst (AndL src1 (XorL src2 m1)));
11607   ins_cost(INSN_COST);
11608   format %{ "bic  $dst, $src1, $src2" %}
11609 
11610   ins_encode %{
11611     __ bic(as_Register($dst$$reg),
11612               as_Register($src1$$reg),
11613               as_Register($src2$$reg),
11614               Assembler::LSL, 0);
11615   %}
11616 
11617   ins_pipe(ialu_reg_reg);
11618 %}
11619 
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622 instruct OrI_reg_not_reg(iRegINoSp dst,
11623                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11624   match(Set dst (OrI src1 (XorI src2 m1)));
11625   ins_cost(INSN_COST);
11626   format %{ "ornw  $dst, $src1, $src2" %}
11627 
11628   ins_encode %{
11629     __ ornw(as_Register($dst$$reg),
11630               as_Register($src1$$reg),
11631               as_Register($src2$$reg),
11632               Assembler::LSL, 0);
11633   %}
11634 
11635   ins_pipe(ialu_reg_reg);
11636 %}
11637 
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 instruct OrL_reg_not_reg(iRegLNoSp dst,
11641                          iRegL src1, iRegL src2, immL_M1 m1) %{
11642   match(Set dst (OrL src1 (XorL src2 m1)));
11643   ins_cost(INSN_COST);
11644   format %{ "orn  $dst, $src1, $src2" %}
11645 
11646   ins_encode %{
11647     __ orn(as_Register($dst$$reg),
11648               as_Register($src1$$reg),
11649               as_Register($src2$$reg),
11650               Assembler::LSL, 0);
11651   %}
11652 
11653   ins_pipe(ialu_reg_reg);
11654 %}
11655 
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658 instruct XorI_reg_not_reg(iRegINoSp dst,
11659                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11660   match(Set dst (XorI m1 (XorI src2 src1)));
11661   ins_cost(INSN_COST);
11662   format %{ "eonw  $dst, $src1, $src2" %}
11663 
11664   ins_encode %{
11665     __ eonw(as_Register($dst$$reg),
11666               as_Register($src1$$reg),
11667               as_Register($src2$$reg),
11668               Assembler::LSL, 0);
11669   %}
11670 
11671   ins_pipe(ialu_reg_reg);
11672 %}
11673 
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 instruct XorL_reg_not_reg(iRegLNoSp dst,
11677                          iRegL src1, iRegL src2, immL_M1 m1) %{
11678   match(Set dst (XorL m1 (XorL src2 src1)));
11679   ins_cost(INSN_COST);
11680   format %{ "eon  $dst, $src1, $src2" %}
11681 
11682   ins_encode %{
11683     __ eon(as_Register($dst$$reg),
11684               as_Register($src1$$reg),
11685               as_Register($src2$$reg),
11686               Assembler::LSL, 0);
11687   %}
11688 
11689   ins_pipe(ialu_reg_reg);
11690 %}
11691 
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 // val & (-1 ^ (val >>> shift)) ==> bicw
11695 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11696                          iRegIorL2I src1, iRegIorL2I src2,
11697                          immI src3, immI_M1 src4) %{
11698   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11699   ins_cost(1.9 * INSN_COST);
11700   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11701 
11702   ins_encode %{
11703     __ bicw(as_Register($dst$$reg),
11704               as_Register($src1$$reg),
11705               as_Register($src2$$reg),
11706               Assembler::LSR,
11707               $src3$$constant & 0x1f);
11708   %}
11709 
11710   ins_pipe(ialu_reg_reg_shift);
11711 %}
11712 
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 // val & (-1 ^ (val >>> shift)) ==> bic
11716 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11717                          iRegL src1, iRegL src2,
11718                          immI src3, immL_M1 src4) %{
11719   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11720   ins_cost(1.9 * INSN_COST);
11721   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11722 
11723   ins_encode %{
11724     __ bic(as_Register($dst$$reg),
11725               as_Register($src1$$reg),
11726               as_Register($src2$$reg),
11727               Assembler::LSR,
11728               $src3$$constant & 0x3f);
11729   %}
11730 
11731   ins_pipe(ialu_reg_reg_shift);
11732 %}
11733 
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 // val & (-1 ^ (val >> shift)) ==> bicw
11737 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11738                          iRegIorL2I src1, iRegIorL2I src2,
11739                          immI src3, immI_M1 src4) %{
11740   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11741   ins_cost(1.9 * INSN_COST);
11742   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11743 
11744   ins_encode %{
11745     __ bicw(as_Register($dst$$reg),
11746               as_Register($src1$$reg),
11747               as_Register($src2$$reg),
11748               Assembler::ASR,
11749               $src3$$constant & 0x1f);
11750   %}
11751 
11752   ins_pipe(ialu_reg_reg_shift);
11753 %}
11754 
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 // val & (-1 ^ (val >> shift)) ==> bic
11758 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11759                          iRegL src1, iRegL src2,
11760                          immI src3, immL_M1 src4) %{
11761   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11762   ins_cost(1.9 * INSN_COST);
11763   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11764 
11765   ins_encode %{
11766     __ bic(as_Register($dst$$reg),
11767               as_Register($src1$$reg),
11768               as_Register($src2$$reg),
11769               Assembler::ASR,
11770               $src3$$constant & 0x3f);
11771   %}
11772 
11773   ins_pipe(ialu_reg_reg_shift);
11774 %}
11775 
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val & (-1 ^ (val ror shift)) ==> bicw
11779 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11780                          iRegIorL2I src1, iRegIorL2I src2,
11781                          immI src3, immI_M1 src4) %{
11782   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11783   ins_cost(1.9 * INSN_COST);
11784   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11785 
11786   ins_encode %{
11787     __ bicw(as_Register($dst$$reg),
11788               as_Register($src1$$reg),
11789               as_Register($src2$$reg),
11790               Assembler::ROR,
11791               $src3$$constant & 0x1f);
11792   %}
11793 
11794   ins_pipe(ialu_reg_reg_shift);
11795 %}
11796 
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val & (-1 ^ (val ror shift)) ==> bic
11800 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11801                          iRegL src1, iRegL src2,
11802                          immI src3, immL_M1 src4) %{
11803   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11804   ins_cost(1.9 * INSN_COST);
11805   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11806 
11807   ins_encode %{
11808     __ bic(as_Register($dst$$reg),
11809               as_Register($src1$$reg),
11810               as_Register($src2$$reg),
11811               Assembler::ROR,
11812               $src3$$constant & 0x3f);
11813   %}
11814 
11815   ins_pipe(ialu_reg_reg_shift);
11816 %}
11817 
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val & (-1 ^ (val << shift)) ==> bicw
11821 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11822                          iRegIorL2I src1, iRegIorL2I src2,
11823                          immI src3, immI_M1 src4) %{
11824   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11825   ins_cost(1.9 * INSN_COST);
11826   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11827 
11828   ins_encode %{
11829     __ bicw(as_Register($dst$$reg),
11830               as_Register($src1$$reg),
11831               as_Register($src2$$reg),
11832               Assembler::LSL,
11833               $src3$$constant & 0x1f);
11834   %}
11835 
11836   ins_pipe(ialu_reg_reg_shift);
11837 %}
11838 
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val & (-1 ^ (val << shift)) ==> bic
11842 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11843                          iRegL src1, iRegL src2,
11844                          immI src3, immL_M1 src4) %{
11845   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11846   ins_cost(1.9 * INSN_COST);
11847   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11848 
11849   ins_encode %{
11850     __ bic(as_Register($dst$$reg),
11851               as_Register($src1$$reg),
11852               as_Register($src2$$reg),
11853               Assembler::LSL,
11854               $src3$$constant & 0x3f);
11855   %}
11856 
11857   ins_pipe(ialu_reg_reg_shift);
11858 %}
11859 
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11863 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11864                          iRegIorL2I src1, iRegIorL2I src2,
11865                          immI src3, immI_M1 src4) %{
11866   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11867   ins_cost(1.9 * INSN_COST);
11868   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11869 
11870   ins_encode %{
11871     __ eonw(as_Register($dst$$reg),
11872               as_Register($src1$$reg),
11873               as_Register($src2$$reg),
11874               Assembler::LSR,
11875               $src3$$constant & 0x1f);
11876   %}
11877 
11878   ins_pipe(ialu_reg_reg_shift);
11879 %}
11880 
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val ^ (-1 ^ (val >>> shift)) ==> eon
11884 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11885                          iRegL src1, iRegL src2,
11886                          immI src3, immL_M1 src4) %{
11887   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11888   ins_cost(1.9 * INSN_COST);
11889   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11890 
11891   ins_encode %{
11892     __ eon(as_Register($dst$$reg),
11893               as_Register($src1$$reg),
11894               as_Register($src2$$reg),
11895               Assembler::LSR,
11896               $src3$$constant & 0x3f);
11897   %}
11898 
11899   ins_pipe(ialu_reg_reg_shift);
11900 %}
11901 
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val ^ (-1 ^ (val >> shift)) ==> eonw
11905 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11906                          iRegIorL2I src1, iRegIorL2I src2,
11907                          immI src3, immI_M1 src4) %{
11908   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11909   ins_cost(1.9 * INSN_COST);
11910   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11911 
11912   ins_encode %{
11913     __ eonw(as_Register($dst$$reg),
11914               as_Register($src1$$reg),
11915               as_Register($src2$$reg),
11916               Assembler::ASR,
11917               $src3$$constant & 0x1f);
11918   %}
11919 
11920   ins_pipe(ialu_reg_reg_shift);
11921 %}
11922 
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val ^ (-1 ^ (val >> shift)) ==> eon
11926 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11927                          iRegL src1, iRegL src2,
11928                          immI src3, immL_M1 src4) %{
11929   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11930   ins_cost(1.9 * INSN_COST);
11931   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11932 
11933   ins_encode %{
11934     __ eon(as_Register($dst$$reg),
11935               as_Register($src1$$reg),
11936               as_Register($src2$$reg),
11937               Assembler::ASR,
11938               $src3$$constant & 0x3f);
11939   %}
11940 
11941   ins_pipe(ialu_reg_reg_shift);
11942 %}
11943 
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 // val ^ (-1 ^ (val ror shift)) ==> eonw
11947 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11948                          iRegIorL2I src1, iRegIorL2I src2,
11949                          immI src3, immI_M1 src4) %{
11950   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11951   ins_cost(1.9 * INSN_COST);
11952   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11953 
11954   ins_encode %{
11955     __ eonw(as_Register($dst$$reg),
11956               as_Register($src1$$reg),
11957               as_Register($src2$$reg),
11958               Assembler::ROR,
11959               $src3$$constant & 0x1f);
11960   %}
11961 
11962   ins_pipe(ialu_reg_reg_shift);
11963 %}
11964 
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 // val ^ (-1 ^ (val ror shift)) ==> eon
11968 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11969                          iRegL src1, iRegL src2,
11970                          immI src3, immL_M1 src4) %{
11971   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11972   ins_cost(1.9 * INSN_COST);
11973   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11974 
11975   ins_encode %{
11976     __ eon(as_Register($dst$$reg),
11977               as_Register($src1$$reg),
11978               as_Register($src2$$reg),
11979               Assembler::ROR,
11980               $src3$$constant & 0x3f);
11981   %}
11982 
11983   ins_pipe(ialu_reg_reg_shift);
11984 %}
11985 
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 // val ^ (-1 ^ (val << shift)) ==> eonw
11989 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11990                          iRegIorL2I src1, iRegIorL2I src2,
11991                          immI src3, immI_M1 src4) %{
11992   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11993   ins_cost(1.9 * INSN_COST);
11994   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11995 
11996   ins_encode %{
11997     __ eonw(as_Register($dst$$reg),
11998               as_Register($src1$$reg),
11999               as_Register($src2$$reg),
12000               Assembler::LSL,
12001               $src3$$constant & 0x1f);
12002   %}
12003 
12004   ins_pipe(ialu_reg_reg_shift);
12005 %}
12006 
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 // val ^ (-1 ^ (val << shift)) ==> eon
12010 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12011                          iRegL src1, iRegL src2,
12012                          immI src3, immL_M1 src4) %{
12013   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12014   ins_cost(1.9 * INSN_COST);
12015   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
12016 
12017   ins_encode %{
12018     __ eon(as_Register($dst$$reg),
12019               as_Register($src1$$reg),
12020               as_Register($src2$$reg),
12021               Assembler::LSL,
12022               $src3$$constant & 0x3f);
12023   %}
12024 
12025   ins_pipe(ialu_reg_reg_shift);
12026 %}
12027 
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 // val | (-1 ^ (val >>> shift)) ==> ornw
12031 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12032                          iRegIorL2I src1, iRegIorL2I src2,
12033                          immI src3, immI_M1 src4) %{
12034   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12035   ins_cost(1.9 * INSN_COST);
12036   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
12037 
12038   ins_encode %{
12039     __ ornw(as_Register($dst$$reg),
12040               as_Register($src1$$reg),
12041               as_Register($src2$$reg),
12042               Assembler::LSR,
12043               $src3$$constant & 0x1f);
12044   %}
12045 
12046   ins_pipe(ialu_reg_reg_shift);
12047 %}
12048 
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 // val | (-1 ^ (val >>> shift)) ==> orn
12052 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12053                          iRegL src1, iRegL src2,
12054                          immI src3, immL_M1 src4) %{
12055   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12056   ins_cost(1.9 * INSN_COST);
12057   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
12058 
12059   ins_encode %{
12060     __ orn(as_Register($dst$$reg),
12061               as_Register($src1$$reg),
12062               as_Register($src2$$reg),
12063               Assembler::LSR,
12064               $src3$$constant & 0x3f);
12065   %}
12066 
12067   ins_pipe(ialu_reg_reg_shift);
12068 %}
12069 
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 // val | (-1 ^ (val >> shift)) ==> ornw
12073 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12074                          iRegIorL2I src1, iRegIorL2I src2,
12075                          immI src3, immI_M1 src4) %{
12076   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12077   ins_cost(1.9 * INSN_COST);
12078   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12079 
12080   ins_encode %{
12081     __ ornw(as_Register($dst$$reg),
12082               as_Register($src1$$reg),
12083               as_Register($src2$$reg),
12084               Assembler::ASR,
12085               $src3$$constant & 0x1f);
12086   %}
12087 
12088   ins_pipe(ialu_reg_reg_shift);
12089 %}
12090 
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 // val | (-1 ^ (val >> shift)) ==> orn
12094 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12095                          iRegL src1, iRegL src2,
12096                          immI src3, immL_M1 src4) %{
12097   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12098   ins_cost(1.9 * INSN_COST);
12099   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12100 
12101   ins_encode %{
12102     __ orn(as_Register($dst$$reg),
12103               as_Register($src1$$reg),
12104               as_Register($src2$$reg),
12105               Assembler::ASR,
12106               $src3$$constant & 0x3f);
12107   %}
12108 
12109   ins_pipe(ialu_reg_reg_shift);
12110 %}
12111 
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 // val | (-1 ^ (val ror shift)) ==> ornw
12115 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12116                          iRegIorL2I src1, iRegIorL2I src2,
12117                          immI src3, immI_M1 src4) %{
12118   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12119   ins_cost(1.9 * INSN_COST);
12120   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12121 
12122   ins_encode %{
12123     __ ornw(as_Register($dst$$reg),
12124               as_Register($src1$$reg),
12125               as_Register($src2$$reg),
12126               Assembler::ROR,
12127               $src3$$constant & 0x1f);
12128   %}
12129 
12130   ins_pipe(ialu_reg_reg_shift);
12131 %}
12132 
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 // val | (-1 ^ (val ror shift)) ==> orn
12136 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12137                          iRegL src1, iRegL src2,
12138                          immI src3, immL_M1 src4) %{
12139   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12140   ins_cost(1.9 * INSN_COST);
12141   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12142 
12143   ins_encode %{
12144     __ orn(as_Register($dst$$reg),
12145               as_Register($src1$$reg),
12146               as_Register($src2$$reg),
12147               Assembler::ROR,
12148               $src3$$constant & 0x3f);
12149   %}
12150 
12151   ins_pipe(ialu_reg_reg_shift);
12152 %}
12153 
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 // val | (-1 ^ (val << shift)) ==> ornw
12157 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12158                          iRegIorL2I src1, iRegIorL2I src2,
12159                          immI src3, immI_M1 src4) %{
12160   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12161   ins_cost(1.9 * INSN_COST);
12162   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12163 
12164   ins_encode %{
12165     __ ornw(as_Register($dst$$reg),
12166               as_Register($src1$$reg),
12167               as_Register($src2$$reg),
12168               Assembler::LSL,
12169               $src3$$constant & 0x1f);
12170   %}
12171 
12172   ins_pipe(ialu_reg_reg_shift);
12173 %}
12174 
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 // val | (-1 ^ (val << shift)) ==> orn
12178 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12179                          iRegL src1, iRegL src2,
12180                          immI src3, immL_M1 src4) %{
12181   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12182   ins_cost(1.9 * INSN_COST);
12183   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12184 
12185   ins_encode %{
12186     __ orn(as_Register($dst$$reg),
12187               as_Register($src1$$reg),
12188               as_Register($src2$$reg),
12189               Assembler::LSL,
12190               $src3$$constant & 0x3f);
12191   %}
12192 
12193   ins_pipe(ialu_reg_reg_shift);
12194 %}
12195 
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12199                          iRegIorL2I src1, iRegIorL2I src2,
12200                          immI src3) %{
12201   match(Set dst (AndI src1 (URShiftI src2 src3)));
12202 
12203   ins_cost(1.9 * INSN_COST);
12204   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12205 
12206   ins_encode %{
12207     __ andw(as_Register($dst$$reg),
12208               as_Register($src1$$reg),
12209               as_Register($src2$$reg),
12210               Assembler::LSR,
12211               $src3$$constant & 0x1f);
12212   %}
12213 
12214   ins_pipe(ialu_reg_reg_shift);
12215 %}
12216 
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12220                          iRegL src1, iRegL src2,
12221                          immI src3) %{
12222   match(Set dst (AndL src1 (URShiftL src2 src3)));
12223 
12224   ins_cost(1.9 * INSN_COST);
12225   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12226 
12227   ins_encode %{
12228     __ andr(as_Register($dst$$reg),
12229               as_Register($src1$$reg),
12230               as_Register($src2$$reg),
12231               Assembler::LSR,
12232               $src3$$constant & 0x3f);
12233   %}
12234 
12235   ins_pipe(ialu_reg_reg_shift);
12236 %}
12237 
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12241                          iRegIorL2I src1, iRegIorL2I src2,
12242                          immI src3) %{
12243   match(Set dst (AndI src1 (RShiftI src2 src3)));
12244 
12245   ins_cost(1.9 * INSN_COST);
12246   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12247 
12248   ins_encode %{
12249     __ andw(as_Register($dst$$reg),
12250               as_Register($src1$$reg),
12251               as_Register($src2$$reg),
12252               Assembler::ASR,
12253               $src3$$constant & 0x1f);
12254   %}
12255 
12256   ins_pipe(ialu_reg_reg_shift);
12257 %}
12258 
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12262                          iRegL src1, iRegL src2,
12263                          immI src3) %{
12264   match(Set dst (AndL src1 (RShiftL src2 src3)));
12265 
12266   ins_cost(1.9 * INSN_COST);
12267   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12268 
12269   ins_encode %{
12270     __ andr(as_Register($dst$$reg),
12271               as_Register($src1$$reg),
12272               as_Register($src2$$reg),
12273               Assembler::ASR,
12274               $src3$$constant & 0x3f);
12275   %}
12276 
12277   ins_pipe(ialu_reg_reg_shift);
12278 %}
12279 
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12283                          iRegIorL2I src1, iRegIorL2I src2,
12284                          immI src3) %{
12285   match(Set dst (AndI src1 (LShiftI src2 src3)));
12286 
12287   ins_cost(1.9 * INSN_COST);
12288   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12289 
12290   ins_encode %{
12291     __ andw(as_Register($dst$$reg),
12292               as_Register($src1$$reg),
12293               as_Register($src2$$reg),
12294               Assembler::LSL,
12295               $src3$$constant & 0x1f);
12296   %}
12297 
12298   ins_pipe(ialu_reg_reg_shift);
12299 %}
12300 
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12304                          iRegL src1, iRegL src2,
12305                          immI src3) %{
12306   match(Set dst (AndL src1 (LShiftL src2 src3)));
12307 
12308   ins_cost(1.9 * INSN_COST);
12309   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12310 
12311   ins_encode %{
12312     __ andr(as_Register($dst$$reg),
12313               as_Register($src1$$reg),
12314               as_Register($src2$$reg),
12315               Assembler::LSL,
12316               $src3$$constant & 0x3f);
12317   %}
12318 
12319   ins_pipe(ialu_reg_reg_shift);
12320 %}
12321 
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12325                          iRegIorL2I src1, iRegIorL2I src2,
12326                          immI src3) %{
12327   match(Set dst (AndI src1 (RotateRight src2 src3)));
12328 
12329   ins_cost(1.9 * INSN_COST);
12330   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12331 
12332   ins_encode %{
12333     __ andw(as_Register($dst$$reg),
12334               as_Register($src1$$reg),
12335               as_Register($src2$$reg),
12336               Assembler::ROR,
12337               $src3$$constant & 0x1f);
12338   %}
12339 
12340   ins_pipe(ialu_reg_reg_shift);
12341 %}
12342 
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12346                          iRegL src1, iRegL src2,
12347                          immI src3) %{
12348   match(Set dst (AndL src1 (RotateRight src2 src3)));
12349 
12350   ins_cost(1.9 * INSN_COST);
12351   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12352 
12353   ins_encode %{
12354     __ andr(as_Register($dst$$reg),
12355               as_Register($src1$$reg),
12356               as_Register($src2$$reg),
12357               Assembler::ROR,
12358               $src3$$constant & 0x3f);
12359   %}
12360 
12361   ins_pipe(ialu_reg_reg_shift);
12362 %}
12363 
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12367                          iRegIorL2I src1, iRegIorL2I src2,
12368                          immI src3) %{
12369   match(Set dst (XorI src1 (URShiftI src2 src3)));
12370 
12371   ins_cost(1.9 * INSN_COST);
12372   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12373 
12374   ins_encode %{
12375     __ eorw(as_Register($dst$$reg),
12376               as_Register($src1$$reg),
12377               as_Register($src2$$reg),
12378               Assembler::LSR,
12379               $src3$$constant & 0x1f);
12380   %}
12381 
12382   ins_pipe(ialu_reg_reg_shift);
12383 %}
12384 
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12388                          iRegL src1, iRegL src2,
12389                          immI src3) %{
12390   match(Set dst (XorL src1 (URShiftL src2 src3)));
12391 
12392   ins_cost(1.9 * INSN_COST);
12393   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12394 
12395   ins_encode %{
12396     __ eor(as_Register($dst$$reg),
12397               as_Register($src1$$reg),
12398               as_Register($src2$$reg),
12399               Assembler::LSR,
12400               $src3$$constant & 0x3f);
12401   %}
12402 
12403   ins_pipe(ialu_reg_reg_shift);
12404 %}
12405 
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12409                          iRegIorL2I src1, iRegIorL2I src2,
12410                          immI src3) %{
12411   match(Set dst (XorI src1 (RShiftI src2 src3)));
12412 
12413   ins_cost(1.9 * INSN_COST);
12414   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12415 
12416   ins_encode %{
12417     __ eorw(as_Register($dst$$reg),
12418               as_Register($src1$$reg),
12419               as_Register($src2$$reg),
12420               Assembler::ASR,
12421               $src3$$constant & 0x1f);
12422   %}
12423 
12424   ins_pipe(ialu_reg_reg_shift);
12425 %}
12426 
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12430                          iRegL src1, iRegL src2,
12431                          immI src3) %{
12432   match(Set dst (XorL src1 (RShiftL src2 src3)));
12433 
12434   ins_cost(1.9 * INSN_COST);
12435   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12436 
12437   ins_encode %{
12438     __ eor(as_Register($dst$$reg),
12439               as_Register($src1$$reg),
12440               as_Register($src2$$reg),
12441               Assembler::ASR,
12442               $src3$$constant & 0x3f);
12443   %}
12444 
12445   ins_pipe(ialu_reg_reg_shift);
12446 %}
12447 
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12451                          iRegIorL2I src1, iRegIorL2I src2,
12452                          immI src3) %{
12453   match(Set dst (XorI src1 (LShiftI src2 src3)));
12454 
12455   ins_cost(1.9 * INSN_COST);
12456   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12457 
12458   ins_encode %{
12459     __ eorw(as_Register($dst$$reg),
12460               as_Register($src1$$reg),
12461               as_Register($src2$$reg),
12462               Assembler::LSL,
12463               $src3$$constant & 0x1f);
12464   %}
12465 
12466   ins_pipe(ialu_reg_reg_shift);
12467 %}
12468 
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12472                          iRegL src1, iRegL src2,
12473                          immI src3) %{
12474   match(Set dst (XorL src1 (LShiftL src2 src3)));
12475 
12476   ins_cost(1.9 * INSN_COST);
12477   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12478 
12479   ins_encode %{
12480     __ eor(as_Register($dst$$reg),
12481               as_Register($src1$$reg),
12482               as_Register($src2$$reg),
12483               Assembler::LSL,
12484               $src3$$constant & 0x3f);
12485   %}
12486 
12487   ins_pipe(ialu_reg_reg_shift);
12488 %}
12489 
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12493                          iRegIorL2I src1, iRegIorL2I src2,
12494                          immI src3) %{
12495   match(Set dst (XorI src1 (RotateRight src2 src3)));
12496 
12497   ins_cost(1.9 * INSN_COST);
12498   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12499 
12500   ins_encode %{
12501     __ eorw(as_Register($dst$$reg),
12502               as_Register($src1$$reg),
12503               as_Register($src2$$reg),
12504               Assembler::ROR,
12505               $src3$$constant & 0x1f);
12506   %}
12507 
12508   ins_pipe(ialu_reg_reg_shift);
12509 %}
12510 
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12514                          iRegL src1, iRegL src2,
12515                          immI src3) %{
12516   match(Set dst (XorL src1 (RotateRight src2 src3)));
12517 
12518   ins_cost(1.9 * INSN_COST);
12519   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12520 
12521   ins_encode %{
12522     __ eor(as_Register($dst$$reg),
12523               as_Register($src1$$reg),
12524               as_Register($src2$$reg),
12525               Assembler::ROR,
12526               $src3$$constant & 0x3f);
12527   %}
12528 
12529   ins_pipe(ialu_reg_reg_shift);
12530 %}
12531 
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12535                          iRegIorL2I src1, iRegIorL2I src2,
12536                          immI src3) %{
12537   match(Set dst (OrI src1 (URShiftI src2 src3)));
12538 
12539   ins_cost(1.9 * INSN_COST);
12540   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12541 
12542   ins_encode %{
12543     __ orrw(as_Register($dst$$reg),
12544               as_Register($src1$$reg),
12545               as_Register($src2$$reg),
12546               Assembler::LSR,
12547               $src3$$constant & 0x1f);
12548   %}
12549 
12550   ins_pipe(ialu_reg_reg_shift);
12551 %}
12552 
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12556                          iRegL src1, iRegL src2,
12557                          immI src3) %{
12558   match(Set dst (OrL src1 (URShiftL src2 src3)));
12559 
12560   ins_cost(1.9 * INSN_COST);
12561   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12562 
12563   ins_encode %{
12564     __ orr(as_Register($dst$$reg),
12565               as_Register($src1$$reg),
12566               as_Register($src2$$reg),
12567               Assembler::LSR,
12568               $src3$$constant & 0x3f);
12569   %}
12570 
12571   ins_pipe(ialu_reg_reg_shift);
12572 %}
12573 
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12577                          iRegIorL2I src1, iRegIorL2I src2,
12578                          immI src3) %{
12579   match(Set dst (OrI src1 (RShiftI src2 src3)));
12580 
12581   ins_cost(1.9 * INSN_COST);
12582   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12583 
12584   ins_encode %{
12585     __ orrw(as_Register($dst$$reg),
12586               as_Register($src1$$reg),
12587               as_Register($src2$$reg),
12588               Assembler::ASR,
12589               $src3$$constant & 0x1f);
12590   %}
12591 
12592   ins_pipe(ialu_reg_reg_shift);
12593 %}
12594 
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12598                          iRegL src1, iRegL src2,
12599                          immI src3) %{
12600   match(Set dst (OrL src1 (RShiftL src2 src3)));
12601 
12602   ins_cost(1.9 * INSN_COST);
12603   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12604 
12605   ins_encode %{
12606     __ orr(as_Register($dst$$reg),
12607               as_Register($src1$$reg),
12608               as_Register($src2$$reg),
12609               Assembler::ASR,
12610               $src3$$constant & 0x3f);
12611   %}
12612 
12613   ins_pipe(ialu_reg_reg_shift);
12614 %}
12615 
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12619                          iRegIorL2I src1, iRegIorL2I src2,
12620                          immI src3) %{
12621   match(Set dst (OrI src1 (LShiftI src2 src3)));
12622 
12623   ins_cost(1.9 * INSN_COST);
12624   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12625 
12626   ins_encode %{
12627     __ orrw(as_Register($dst$$reg),
12628               as_Register($src1$$reg),
12629               as_Register($src2$$reg),
12630               Assembler::LSL,
12631               $src3$$constant & 0x1f);
12632   %}
12633 
12634   ins_pipe(ialu_reg_reg_shift);
12635 %}
12636 
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12640                          iRegL src1, iRegL src2,
12641                          immI src3) %{
12642   match(Set dst (OrL src1 (LShiftL src2 src3)));
12643 
12644   ins_cost(1.9 * INSN_COST);
12645   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12646 
12647   ins_encode %{
12648     __ orr(as_Register($dst$$reg),
12649               as_Register($src1$$reg),
12650               as_Register($src2$$reg),
12651               Assembler::LSL,
12652               $src3$$constant & 0x3f);
12653   %}
12654 
12655   ins_pipe(ialu_reg_reg_shift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12661                          iRegIorL2I src1, iRegIorL2I src2,
12662                          immI src3) %{
12663   match(Set dst (OrI src1 (RotateRight src2 src3)));
12664 
12665   ins_cost(1.9 * INSN_COST);
12666   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12667 
12668   ins_encode %{
12669     __ orrw(as_Register($dst$$reg),
12670               as_Register($src1$$reg),
12671               as_Register($src2$$reg),
12672               Assembler::ROR,
12673               $src3$$constant & 0x1f);
12674   %}
12675 
12676   ins_pipe(ialu_reg_reg_shift);
12677 %}
12678 
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12682                          iRegL src1, iRegL src2,
12683                          immI src3) %{
12684   match(Set dst (OrL src1 (RotateRight src2 src3)));
12685 
12686   ins_cost(1.9 * INSN_COST);
12687   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12688 
12689   ins_encode %{
12690     __ orr(as_Register($dst$$reg),
12691               as_Register($src1$$reg),
12692               as_Register($src2$$reg),
12693               Assembler::ROR,
12694               $src3$$constant & 0x3f);
12695   %}
12696 
12697   ins_pipe(ialu_reg_reg_shift);
12698 %}
12699 
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12703                          iRegIorL2I src1, iRegIorL2I src2,
12704                          immI src3) %{
12705   match(Set dst (AddI src1 (URShiftI src2 src3)));
12706 
12707   ins_cost(1.9 * INSN_COST);
12708   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12709 
12710   ins_encode %{
12711     __ addw(as_Register($dst$$reg),
12712               as_Register($src1$$reg),
12713               as_Register($src2$$reg),
12714               Assembler::LSR,
12715               $src3$$constant & 0x1f);
12716   %}
12717 
12718   ins_pipe(ialu_reg_reg_shift);
12719 %}
12720 
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12724                          iRegL src1, iRegL src2,
12725                          immI src3) %{
12726   match(Set dst (AddL src1 (URShiftL src2 src3)));
12727 
12728   ins_cost(1.9 * INSN_COST);
12729   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12730 
12731   ins_encode %{
12732     __ add(as_Register($dst$$reg),
12733               as_Register($src1$$reg),
12734               as_Register($src2$$reg),
12735               Assembler::LSR,
12736               $src3$$constant & 0x3f);
12737   %}
12738 
12739   ins_pipe(ialu_reg_reg_shift);
12740 %}
12741 
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12745                          iRegIorL2I src1, iRegIorL2I src2,
12746                          immI src3) %{
12747   match(Set dst (AddI src1 (RShiftI src2 src3)));
12748 
12749   ins_cost(1.9 * INSN_COST);
12750   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12751 
12752   ins_encode %{
12753     __ addw(as_Register($dst$$reg),
12754               as_Register($src1$$reg),
12755               as_Register($src2$$reg),
12756               Assembler::ASR,
12757               $src3$$constant & 0x1f);
12758   %}
12759 
12760   ins_pipe(ialu_reg_reg_shift);
12761 %}
12762 
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12766                          iRegL src1, iRegL src2,
12767                          immI src3) %{
12768   match(Set dst (AddL src1 (RShiftL src2 src3)));
12769 
12770   ins_cost(1.9 * INSN_COST);
12771   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12772 
12773   ins_encode %{
12774     __ add(as_Register($dst$$reg),
12775               as_Register($src1$$reg),
12776               as_Register($src2$$reg),
12777               Assembler::ASR,
12778               $src3$$constant & 0x3f);
12779   %}
12780 
12781   ins_pipe(ialu_reg_reg_shift);
12782 %}
12783 
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12787                          iRegIorL2I src1, iRegIorL2I src2,
12788                          immI src3) %{
12789   match(Set dst (AddI src1 (LShiftI src2 src3)));
12790 
12791   ins_cost(1.9 * INSN_COST);
12792   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12793 
12794   ins_encode %{
12795     __ addw(as_Register($dst$$reg),
12796               as_Register($src1$$reg),
12797               as_Register($src2$$reg),
12798               Assembler::LSL,
12799               $src3$$constant & 0x1f);
12800   %}
12801 
12802   ins_pipe(ialu_reg_reg_shift);
12803 %}
12804 
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12808                          iRegL src1, iRegL src2,
12809                          immI src3) %{
12810   match(Set dst (AddL src1 (LShiftL src2 src3)));
12811 
12812   ins_cost(1.9 * INSN_COST);
12813   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12814 
12815   ins_encode %{
12816     __ add(as_Register($dst$$reg),
12817               as_Register($src1$$reg),
12818               as_Register($src2$$reg),
12819               Assembler::LSL,
12820               $src3$$constant & 0x3f);
12821   %}
12822 
12823   ins_pipe(ialu_reg_reg_shift);
12824 %}
12825 
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12829                          iRegIorL2I src1, iRegIorL2I src2,
12830                          immI src3) %{
12831   match(Set dst (SubI src1 (URShiftI src2 src3)));
12832 
12833   ins_cost(1.9 * INSN_COST);
12834   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12835 
12836   ins_encode %{
12837     __ subw(as_Register($dst$$reg),
12838               as_Register($src1$$reg),
12839               as_Register($src2$$reg),
12840               Assembler::LSR,
12841               $src3$$constant & 0x1f);
12842   %}
12843 
12844   ins_pipe(ialu_reg_reg_shift);
12845 %}
12846 
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12850                          iRegL src1, iRegL src2,
12851                          immI src3) %{
12852   match(Set dst (SubL src1 (URShiftL src2 src3)));
12853 
12854   ins_cost(1.9 * INSN_COST);
12855   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12856 
12857   ins_encode %{
12858     __ sub(as_Register($dst$$reg),
12859               as_Register($src1$$reg),
12860               as_Register($src2$$reg),
12861               Assembler::LSR,
12862               $src3$$constant & 0x3f);
12863   %}
12864 
12865   ins_pipe(ialu_reg_reg_shift);
12866 %}
12867 
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12871                          iRegIorL2I src1, iRegIorL2I src2,
12872                          immI src3) %{
12873   match(Set dst (SubI src1 (RShiftI src2 src3)));
12874 
12875   ins_cost(1.9 * INSN_COST);
12876   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12877 
12878   ins_encode %{
12879     __ subw(as_Register($dst$$reg),
12880               as_Register($src1$$reg),
12881               as_Register($src2$$reg),
12882               Assembler::ASR,
12883               $src3$$constant & 0x1f);
12884   %}
12885 
12886   ins_pipe(ialu_reg_reg_shift);
12887 %}
12888 
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12892                          iRegL src1, iRegL src2,
12893                          immI src3) %{
12894   match(Set dst (SubL src1 (RShiftL src2 src3)));
12895 
12896   ins_cost(1.9 * INSN_COST);
12897   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12898 
12899   ins_encode %{
12900     __ sub(as_Register($dst$$reg),
12901               as_Register($src1$$reg),
12902               as_Register($src2$$reg),
12903               Assembler::ASR,
12904               $src3$$constant & 0x3f);
12905   %}
12906 
12907   ins_pipe(ialu_reg_reg_shift);
12908 %}
12909 
12910 // This pattern is automatically generated from aarch64_ad.m4
12911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12912 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12913                          iRegIorL2I src1, iRegIorL2I src2,
12914                          immI src3) %{
12915   match(Set dst (SubI src1 (LShiftI src2 src3)));
12916 
12917   ins_cost(1.9 * INSN_COST);
12918   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12919 
12920   ins_encode %{
12921     __ subw(as_Register($dst$$reg),
12922               as_Register($src1$$reg),
12923               as_Register($src2$$reg),
12924               Assembler::LSL,
12925               $src3$$constant & 0x1f);
12926   %}
12927 
12928   ins_pipe(ialu_reg_reg_shift);
12929 %}
12930 
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12934                          iRegL src1, iRegL src2,
12935                          immI src3) %{
12936   match(Set dst (SubL src1 (LShiftL src2 src3)));
12937 
12938   ins_cost(1.9 * INSN_COST);
12939   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12940 
12941   ins_encode %{
12942     __ sub(as_Register($dst$$reg),
12943               as_Register($src1$$reg),
12944               as_Register($src2$$reg),
12945               Assembler::LSL,
12946               $src3$$constant & 0x3f);
12947   %}
12948 
12949   ins_pipe(ialu_reg_reg_shift);
12950 %}
12951 
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 
12955 // Shift Left followed by Shift Right.
12956 // This idiom is used by the compiler for the i2b bytecode etc.
12957 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12958 %{
12959   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12960   ins_cost(INSN_COST * 2);
12961   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12962   ins_encode %{
12963     int lshift = $lshift_count$$constant & 63;
12964     int rshift = $rshift_count$$constant & 63;
12965     int s = 63 - lshift;
12966     int r = (rshift - lshift) & 63;
12967     __ sbfm(as_Register($dst$$reg),
12968             as_Register($src$$reg),
12969             r, s);
12970   %}
12971 
12972   ins_pipe(ialu_reg_shift);
12973 %}
12974 
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 
12978 // Shift Left followed by Shift Right.
12979 // This idiom is used by the compiler for the i2b bytecode etc.
12980 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12981 %{
12982   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12983   ins_cost(INSN_COST * 2);
12984   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12985   ins_encode %{
12986     int lshift = $lshift_count$$constant & 31;
12987     int rshift = $rshift_count$$constant & 31;
12988     int s = 31 - lshift;
12989     int r = (rshift - lshift) & 31;
12990     __ sbfmw(as_Register($dst$$reg),
12991             as_Register($src$$reg),
12992             r, s);
12993   %}
12994 
12995   ins_pipe(ialu_reg_shift);
12996 %}
12997 
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000 
13001 // Shift Left followed by Shift Right.
13002 // This idiom is used by the compiler for the i2b bytecode etc.
13003 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13004 %{
13005   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13006   ins_cost(INSN_COST * 2);
13007   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13008   ins_encode %{
13009     int lshift = $lshift_count$$constant & 63;
13010     int rshift = $rshift_count$$constant & 63;
13011     int s = 63 - lshift;
13012     int r = (rshift - lshift) & 63;
13013     __ ubfm(as_Register($dst$$reg),
13014             as_Register($src$$reg),
13015             r, s);
13016   %}
13017 
13018   ins_pipe(ialu_reg_shift);
13019 %}
13020 
13021 // This pattern is automatically generated from aarch64_ad.m4
13022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13023 
13024 // Shift Left followed by Shift Right.
13025 // This idiom is used by the compiler for the i2b bytecode etc.
13026 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13027 %{
13028   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13029   ins_cost(INSN_COST * 2);
13030   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13031   ins_encode %{
13032     int lshift = $lshift_count$$constant & 31;
13033     int rshift = $rshift_count$$constant & 31;
13034     int s = 31 - lshift;
13035     int r = (rshift - lshift) & 31;
13036     __ ubfmw(as_Register($dst$$reg),
13037             as_Register($src$$reg),
13038             r, s);
13039   %}
13040 
13041   ins_pipe(ialu_reg_shift);
13042 %}
13043 
13044 // Bitfield extract with shift & mask
13045 
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13049 %{
13050   match(Set dst (AndI (URShiftI src rshift) mask));
13051   // Make sure we are not going to exceed what ubfxw can do.
13052   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13053 
13054   ins_cost(INSN_COST);
13055   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13056   ins_encode %{
13057     int rshift = $rshift$$constant & 31;
13058     intptr_t mask = $mask$$constant;
13059     int width = exact_log2(mask+1);
13060     __ ubfxw(as_Register($dst$$reg),
13061             as_Register($src$$reg), rshift, width);
13062   %}
13063   ins_pipe(ialu_reg_shift);
13064 %}
13065 
13066 // This pattern is automatically generated from aarch64_ad.m4
13067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13068 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13069 %{
13070   match(Set dst (AndL (URShiftL src rshift) mask));
13071   // Make sure we are not going to exceed what ubfx can do.
13072   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13073 
13074   ins_cost(INSN_COST);
13075   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13076   ins_encode %{
13077     int rshift = $rshift$$constant & 63;
13078     intptr_t mask = $mask$$constant;
13079     int width = exact_log2_long(mask+1);
13080     __ ubfx(as_Register($dst$$reg),
13081             as_Register($src$$reg), rshift, width);
13082   %}
13083   ins_pipe(ialu_reg_shift);
13084 %}
13085 
13086 
13087 // This pattern is automatically generated from aarch64_ad.m4
13088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13089 
13090 // We can use ubfx when extending an And with a mask when we know mask
13091 // is positive.  We know that because immI_bitmask guarantees it.
13092 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13093 %{
13094   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13095   // Make sure we are not going to exceed what ubfxw can do.
13096   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13097 
13098   ins_cost(INSN_COST * 2);
13099   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13100   ins_encode %{
13101     int rshift = $rshift$$constant & 31;
13102     intptr_t mask = $mask$$constant;
13103     int width = exact_log2(mask+1);
13104     __ ubfx(as_Register($dst$$reg),
13105             as_Register($src$$reg), rshift, width);
13106   %}
13107   ins_pipe(ialu_reg_shift);
13108 %}
13109 
13110 
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 
13114 // We can use ubfiz when masking by a positive number and then left shifting the result.
13115 // We know that the mask is positive because immI_bitmask guarantees it.
13116 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13117 %{
13118   match(Set dst (LShiftI (AndI src mask) lshift));
13119   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13120 
13121   ins_cost(INSN_COST);
13122   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13123   ins_encode %{
13124     int lshift = $lshift$$constant & 31;
13125     intptr_t mask = $mask$$constant;
13126     int width = exact_log2(mask+1);
13127     __ ubfizw(as_Register($dst$$reg),
13128           as_Register($src$$reg), lshift, width);
13129   %}
13130   ins_pipe(ialu_reg_shift);
13131 %}
13132 
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135 
13136 // We can use ubfiz when masking by a positive number and then left shifting the result.
13137 // We know that the mask is positive because immL_bitmask guarantees it.
13138 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13139 %{
13140   match(Set dst (LShiftL (AndL src mask) lshift));
13141   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13142 
13143   ins_cost(INSN_COST);
13144   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13145   ins_encode %{
13146     int lshift = $lshift$$constant & 63;
13147     intptr_t mask = $mask$$constant;
13148     int width = exact_log2_long(mask+1);
13149     __ ubfiz(as_Register($dst$$reg),
13150           as_Register($src$$reg), lshift, width);
13151   %}
13152   ins_pipe(ialu_reg_shift);
13153 %}
13154 
13155 // This pattern is automatically generated from aarch64_ad.m4
13156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13157 
13158 // We can use ubfiz when masking by a positive number and then left shifting the result.
13159 // We know that the mask is positive because immI_bitmask guarantees it.
13160 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13161 %{
13162   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13163   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13164 
13165   ins_cost(INSN_COST);
13166   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13167   ins_encode %{
13168     int lshift = $lshift$$constant & 31;
13169     intptr_t mask = $mask$$constant;
13170     int width = exact_log2(mask+1);
13171     __ ubfizw(as_Register($dst$$reg),
13172           as_Register($src$$reg), lshift, width);
13173   %}
13174   ins_pipe(ialu_reg_shift);
13175 %}
13176 
13177 // This pattern is automatically generated from aarch64_ad.m4
13178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13179 
13180 // We can use ubfiz when masking by a positive number and then left shifting the result.
13181 // We know that the mask is positive because immL_bitmask guarantees it.
13182 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13183 %{
13184   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13185   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13186 
13187   ins_cost(INSN_COST);
13188   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13189   ins_encode %{
13190     int lshift = $lshift$$constant & 63;
13191     intptr_t mask = $mask$$constant;
13192     int width = exact_log2_long(mask+1);
13193     __ ubfiz(as_Register($dst$$reg),
13194           as_Register($src$$reg), lshift, width);
13195   %}
13196   ins_pipe(ialu_reg_shift);
13197 %}
13198 
13199 
13200 // This pattern is automatically generated from aarch64_ad.m4
13201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13202 
13203 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13204 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13205 %{
13206   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13207   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13208 
13209   ins_cost(INSN_COST);
13210   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13211   ins_encode %{
13212     int lshift = $lshift$$constant & 63;
13213     intptr_t mask = $mask$$constant;
13214     int width = exact_log2(mask+1);
13215     __ ubfiz(as_Register($dst$$reg),
13216              as_Register($src$$reg), lshift, width);
13217   %}
13218   ins_pipe(ialu_reg_shift);
13219 %}
13220 
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 
13224 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13225 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13226 %{
13227   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13228   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13229 
13230   ins_cost(INSN_COST);
13231   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13232   ins_encode %{
13233     int lshift = $lshift$$constant & 31;
13234     intptr_t mask = $mask$$constant;
13235     int width = exact_log2(mask+1);
13236     __ ubfiz(as_Register($dst$$reg),
13237              as_Register($src$$reg), lshift, width);
13238   %}
13239   ins_pipe(ialu_reg_shift);
13240 %}
13241 
13242 // This pattern is automatically generated from aarch64_ad.m4
13243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13244 
13245 // Can skip int2long conversions after AND with small bitmask
13246 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13247 %{
13248   match(Set dst (ConvI2L (AndI src msk)));
13249   ins_cost(INSN_COST);
13250   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13251   ins_encode %{
13252     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13253   %}
13254   ins_pipe(ialu_reg_shift);
13255 %}
13256 
13257 
13258 // Rotations
13259 // This pattern is automatically generated from aarch64_ad.m4
13260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13261 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13262 %{
13263   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13264   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13265 
13266   ins_cost(INSN_COST);
13267   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13268 
13269   ins_encode %{
13270     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13271             $rshift$$constant & 63);
13272   %}
13273   ins_pipe(ialu_reg_reg_extr);
13274 %}
13275 
13276 
13277 // This pattern is automatically generated from aarch64_ad.m4
13278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13279 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13280 %{
13281   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13282   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13283 
13284   ins_cost(INSN_COST);
13285   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13286 
13287   ins_encode %{
13288     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13289             $rshift$$constant & 31);
13290   %}
13291   ins_pipe(ialu_reg_reg_extr);
13292 %}
13293 
13294 
13295 // This pattern is automatically generated from aarch64_ad.m4
13296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13297 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13298 %{
13299   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13300   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13301 
13302   ins_cost(INSN_COST);
13303   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13304 
13305   ins_encode %{
13306     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13307             $rshift$$constant & 63);
13308   %}
13309   ins_pipe(ialu_reg_reg_extr);
13310 %}
13311 
13312 
13313 // This pattern is automatically generated from aarch64_ad.m4
13314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13315 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13316 %{
13317   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13318   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13319 
13320   ins_cost(INSN_COST);
13321   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13322 
13323   ins_encode %{
13324     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13325             $rshift$$constant & 31);
13326   %}
13327   ins_pipe(ialu_reg_reg_extr);
13328 %}
13329 
13330 
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13334 %{
13335   match(Set dst (RotateRight src shift));
13336 
13337   ins_cost(INSN_COST);
13338   format %{ "ror    $dst, $src, $shift" %}
13339 
13340   ins_encode %{
13341      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13342                $shift$$constant & 0x1f);
13343   %}
13344   ins_pipe(ialu_reg_reg_vshift);
13345 %}
13346 
13347 // This pattern is automatically generated from aarch64_ad.m4
13348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13349 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13350 %{
13351   match(Set dst (RotateRight src shift));
13352 
13353   ins_cost(INSN_COST);
13354   format %{ "ror    $dst, $src, $shift" %}
13355 
13356   ins_encode %{
13357      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13358                $shift$$constant & 0x3f);
13359   %}
13360   ins_pipe(ialu_reg_reg_vshift);
13361 %}
13362 
13363 // This pattern is automatically generated from aarch64_ad.m4
13364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13365 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13366 %{
13367   match(Set dst (RotateRight src shift));
13368 
13369   ins_cost(INSN_COST);
13370   format %{ "ror    $dst, $src, $shift" %}
13371 
13372   ins_encode %{
13373      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13374   %}
13375   ins_pipe(ialu_reg_reg_vshift);
13376 %}
13377 
13378 // This pattern is automatically generated from aarch64_ad.m4
13379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13380 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13381 %{
13382   match(Set dst (RotateRight src shift));
13383 
13384   ins_cost(INSN_COST);
13385   format %{ "ror    $dst, $src, $shift" %}
13386 
13387   ins_encode %{
13388      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13389   %}
13390   ins_pipe(ialu_reg_reg_vshift);
13391 %}
13392 
13393 // This pattern is automatically generated from aarch64_ad.m4
13394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13395 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13396 %{
13397   match(Set dst (RotateLeft src shift));
13398 
13399   ins_cost(INSN_COST);
13400   format %{ "rol    $dst, $src, $shift" %}
13401 
13402   ins_encode %{
13403      __ subw(rscratch1, zr, as_Register($shift$$reg));
13404      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13405   %}
13406   ins_pipe(ialu_reg_reg_vshift);
13407 %}
13408 
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13412 %{
13413   match(Set dst (RotateLeft src shift));
13414 
13415   ins_cost(INSN_COST);
13416   format %{ "rol    $dst, $src, $shift" %}
13417 
13418   ins_encode %{
13419      __ subw(rscratch1, zr, as_Register($shift$$reg));
13420      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13421   %}
13422   ins_pipe(ialu_reg_reg_vshift);
13423 %}
13424 
13425 
13426 // Add/subtract (extended)
13427 
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13431 %{
13432   match(Set dst (AddL src1 (ConvI2L src2)));
13433   ins_cost(INSN_COST);
13434   format %{ "add  $dst, $src1, $src2, sxtw" %}
13435 
13436    ins_encode %{
13437      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13438             as_Register($src2$$reg), ext::sxtw);
13439    %}
13440   ins_pipe(ialu_reg_reg);
13441 %}
13442 
13443 // This pattern is automatically generated from aarch64_ad.m4
13444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13445 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13446 %{
13447   match(Set dst (SubL src1 (ConvI2L src2)));
13448   ins_cost(INSN_COST);
13449   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13450 
13451    ins_encode %{
13452      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13453             as_Register($src2$$reg), ext::sxtw);
13454    %}
13455   ins_pipe(ialu_reg_reg);
13456 %}
13457 
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13461 %{
13462   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13463   ins_cost(INSN_COST);
13464   format %{ "add  $dst, $src1, $src2, sxth" %}
13465 
13466    ins_encode %{
13467      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13468             as_Register($src2$$reg), ext::sxth);
13469    %}
13470   ins_pipe(ialu_reg_reg);
13471 %}
13472 
13473 // This pattern is automatically generated from aarch64_ad.m4
13474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13475 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13476 %{
13477   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13478   ins_cost(INSN_COST);
13479   format %{ "add  $dst, $src1, $src2, sxtb" %}
13480 
13481    ins_encode %{
13482      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13483             as_Register($src2$$reg), ext::sxtb);
13484    %}
13485   ins_pipe(ialu_reg_reg);
13486 %}
13487 
13488 // This pattern is automatically generated from aarch64_ad.m4
13489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13490 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13491 %{
13492   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13493   ins_cost(INSN_COST);
13494   format %{ "add  $dst, $src1, $src2, uxtb" %}
13495 
13496    ins_encode %{
13497      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13498             as_Register($src2$$reg), ext::uxtb);
13499    %}
13500   ins_pipe(ialu_reg_reg);
13501 %}
13502 
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13506 %{
13507   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13508   ins_cost(INSN_COST);
13509   format %{ "add  $dst, $src1, $src2, sxth" %}
13510 
13511    ins_encode %{
13512      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13513             as_Register($src2$$reg), ext::sxth);
13514    %}
13515   ins_pipe(ialu_reg_reg);
13516 %}
13517 
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13521 %{
13522   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13523   ins_cost(INSN_COST);
13524   format %{ "add  $dst, $src1, $src2, sxtw" %}
13525 
13526    ins_encode %{
13527      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13528             as_Register($src2$$reg), ext::sxtw);
13529    %}
13530   ins_pipe(ialu_reg_reg);
13531 %}
13532 
13533 // This pattern is automatically generated from aarch64_ad.m4
13534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13535 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13536 %{
13537   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13538   ins_cost(INSN_COST);
13539   format %{ "add  $dst, $src1, $src2, sxtb" %}
13540 
13541    ins_encode %{
13542      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13543             as_Register($src2$$reg), ext::sxtb);
13544    %}
13545   ins_pipe(ialu_reg_reg);
13546 %}
13547 
13548 // This pattern is automatically generated from aarch64_ad.m4
13549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13550 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13551 %{
13552   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13553   ins_cost(INSN_COST);
13554   format %{ "add  $dst, $src1, $src2, uxtb" %}
13555 
13556    ins_encode %{
13557      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13558             as_Register($src2$$reg), ext::uxtb);
13559    %}
13560   ins_pipe(ialu_reg_reg);
13561 %}
13562 
13563 // This pattern is automatically generated from aarch64_ad.m4
13564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13565 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13566 %{
13567   match(Set dst (AddI src1 (AndI src2 mask)));
13568   ins_cost(INSN_COST);
13569   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13570 
13571    ins_encode %{
13572      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13573             as_Register($src2$$reg), ext::uxtb);
13574    %}
13575   ins_pipe(ialu_reg_reg);
13576 %}
13577 
13578 // This pattern is automatically generated from aarch64_ad.m4
13579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13580 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13581 %{
13582   match(Set dst (AddI src1 (AndI src2 mask)));
13583   ins_cost(INSN_COST);
13584   format %{ "addw  $dst, $src1, $src2, uxth" %}
13585 
13586    ins_encode %{
13587      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13588             as_Register($src2$$reg), ext::uxth);
13589    %}
13590   ins_pipe(ialu_reg_reg);
13591 %}
13592 
13593 // This pattern is automatically generated from aarch64_ad.m4
13594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13595 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13596 %{
13597   match(Set dst (AddL src1 (AndL src2 mask)));
13598   ins_cost(INSN_COST);
13599   format %{ "add  $dst, $src1, $src2, uxtb" %}
13600 
13601    ins_encode %{
13602      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13603             as_Register($src2$$reg), ext::uxtb);
13604    %}
13605   ins_pipe(ialu_reg_reg);
13606 %}
13607 
13608 // This pattern is automatically generated from aarch64_ad.m4
13609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13610 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13611 %{
13612   match(Set dst (AddL src1 (AndL src2 mask)));
13613   ins_cost(INSN_COST);
13614   format %{ "add  $dst, $src1, $src2, uxth" %}
13615 
13616    ins_encode %{
13617      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13618             as_Register($src2$$reg), ext::uxth);
13619    %}
13620   ins_pipe(ialu_reg_reg);
13621 %}
13622 
13623 // This pattern is automatically generated from aarch64_ad.m4
13624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13625 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13626 %{
13627   match(Set dst (AddL src1 (AndL src2 mask)));
13628   ins_cost(INSN_COST);
13629   format %{ "add  $dst, $src1, $src2, uxtw" %}
13630 
13631    ins_encode %{
13632      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13633             as_Register($src2$$reg), ext::uxtw);
13634    %}
13635   ins_pipe(ialu_reg_reg);
13636 %}
13637 
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13641 %{
13642   match(Set dst (SubI src1 (AndI src2 mask)));
13643   ins_cost(INSN_COST);
13644   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13645 
13646    ins_encode %{
13647      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13648             as_Register($src2$$reg), ext::uxtb);
13649    %}
13650   ins_pipe(ialu_reg_reg);
13651 %}
13652 
13653 // This pattern is automatically generated from aarch64_ad.m4
13654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13655 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13656 %{
13657   match(Set dst (SubI src1 (AndI src2 mask)));
13658   ins_cost(INSN_COST);
13659   format %{ "subw  $dst, $src1, $src2, uxth" %}
13660 
13661    ins_encode %{
13662      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13663             as_Register($src2$$reg), ext::uxth);
13664    %}
13665   ins_pipe(ialu_reg_reg);
13666 %}
13667 
13668 // This pattern is automatically generated from aarch64_ad.m4
13669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13670 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13671 %{
13672   match(Set dst (SubL src1 (AndL src2 mask)));
13673   ins_cost(INSN_COST);
13674   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13675 
13676    ins_encode %{
13677      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13678             as_Register($src2$$reg), ext::uxtb);
13679    %}
13680   ins_pipe(ialu_reg_reg);
13681 %}
13682 
13683 // This pattern is automatically generated from aarch64_ad.m4
13684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13685 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13686 %{
13687   match(Set dst (SubL src1 (AndL src2 mask)));
13688   ins_cost(INSN_COST);
13689   format %{ "sub  $dst, $src1, $src2, uxth" %}
13690 
13691    ins_encode %{
13692      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13693             as_Register($src2$$reg), ext::uxth);
13694    %}
13695   ins_pipe(ialu_reg_reg);
13696 %}
13697 
13698 // This pattern is automatically generated from aarch64_ad.m4
13699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13700 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13701 %{
13702   match(Set dst (SubL src1 (AndL src2 mask)));
13703   ins_cost(INSN_COST);
13704   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13705 
13706    ins_encode %{
13707      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13708             as_Register($src2$$reg), ext::uxtw);
13709    %}
13710   ins_pipe(ialu_reg_reg);
13711 %}
13712 
13713 
13714 // This pattern is automatically generated from aarch64_ad.m4
13715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13716 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13717 %{
13718   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13719   ins_cost(1.9 * INSN_COST);
13720   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13721 
13722    ins_encode %{
13723      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13724             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13725    %}
13726   ins_pipe(ialu_reg_reg_shift);
13727 %}
13728 
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13732 %{
13733   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13734   ins_cost(1.9 * INSN_COST);
13735   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13736 
13737    ins_encode %{
13738      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13739             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13740    %}
13741   ins_pipe(ialu_reg_reg_shift);
13742 %}
13743 
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13747 %{
13748   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13749   ins_cost(1.9 * INSN_COST);
13750   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13751 
13752    ins_encode %{
13753      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13754             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13755    %}
13756   ins_pipe(ialu_reg_reg_shift);
13757 %}
13758 
13759 // This pattern is automatically generated from aarch64_ad.m4
13760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13761 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13762 %{
13763   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13764   ins_cost(1.9 * INSN_COST);
13765   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13766 
13767    ins_encode %{
13768      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13769             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13770    %}
13771   ins_pipe(ialu_reg_reg_shift);
13772 %}
13773 
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13777 %{
13778   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13779   ins_cost(1.9 * INSN_COST);
13780   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13781 
13782    ins_encode %{
13783      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13784             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13785    %}
13786   ins_pipe(ialu_reg_reg_shift);
13787 %}
13788 
13789 // This pattern is automatically generated from aarch64_ad.m4
13790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13791 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13792 %{
13793   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13794   ins_cost(1.9 * INSN_COST);
13795   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13796 
13797    ins_encode %{
13798      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13799             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13800    %}
13801   ins_pipe(ialu_reg_reg_shift);
13802 %}
13803 
13804 // This pattern is automatically generated from aarch64_ad.m4
13805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13806 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13807 %{
13808   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13809   ins_cost(1.9 * INSN_COST);
13810   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13811 
13812    ins_encode %{
13813      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13814             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13815    %}
13816   ins_pipe(ialu_reg_reg_shift);
13817 %}
13818 
13819 // This pattern is automatically generated from aarch64_ad.m4
13820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13821 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13822 %{
13823   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13824   ins_cost(1.9 * INSN_COST);
13825   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13826 
13827    ins_encode %{
13828      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13829             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13830    %}
13831   ins_pipe(ialu_reg_reg_shift);
13832 %}
13833 
13834 // This pattern is automatically generated from aarch64_ad.m4
13835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13836 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13837 %{
13838   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13839   ins_cost(1.9 * INSN_COST);
13840   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13841 
13842    ins_encode %{
13843      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13844             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13845    %}
13846   ins_pipe(ialu_reg_reg_shift);
13847 %}
13848 
13849 // This pattern is automatically generated from aarch64_ad.m4
13850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13851 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13852 %{
13853   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13854   ins_cost(1.9 * INSN_COST);
13855   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13856 
13857    ins_encode %{
13858      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13859             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13860    %}
13861   ins_pipe(ialu_reg_reg_shift);
13862 %}
13863 
13864 // This pattern is automatically generated from aarch64_ad.m4
13865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13866 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13867 %{
13868   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13869   ins_cost(1.9 * INSN_COST);
13870   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13871 
13872    ins_encode %{
13873      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13874             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13875    %}
13876   ins_pipe(ialu_reg_reg_shift);
13877 %}
13878 
13879 // This pattern is automatically generated from aarch64_ad.m4
13880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13881 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13882 %{
13883   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13884   ins_cost(1.9 * INSN_COST);
13885   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13886 
13887    ins_encode %{
13888      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13889             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13890    %}
13891   ins_pipe(ialu_reg_reg_shift);
13892 %}
13893 
13894 // This pattern is automatically generated from aarch64_ad.m4
13895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13896 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13897 %{
13898   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13899   ins_cost(1.9 * INSN_COST);
13900   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13901 
13902    ins_encode %{
13903      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13904             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13905    %}
13906   ins_pipe(ialu_reg_reg_shift);
13907 %}
13908 
13909 // This pattern is automatically generated from aarch64_ad.m4
13910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13911 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13912 %{
13913   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13914   ins_cost(1.9 * INSN_COST);
13915   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13916 
13917    ins_encode %{
13918      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13919             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13920    %}
13921   ins_pipe(ialu_reg_reg_shift);
13922 %}
13923 
13924 // This pattern is automatically generated from aarch64_ad.m4
13925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13926 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13927 %{
13928   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13929   ins_cost(1.9 * INSN_COST);
13930   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13931 
13932    ins_encode %{
13933      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13934             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13935    %}
13936   ins_pipe(ialu_reg_reg_shift);
13937 %}
13938 
13939 // This pattern is automatically generated from aarch64_ad.m4
13940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13941 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13942 %{
13943   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13944   ins_cost(1.9 * INSN_COST);
13945   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13946 
13947    ins_encode %{
13948      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13949             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13950    %}
13951   ins_pipe(ialu_reg_reg_shift);
13952 %}
13953 
13954 // This pattern is automatically generated from aarch64_ad.m4
13955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13956 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13957 %{
13958   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13959   ins_cost(1.9 * INSN_COST);
13960   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13961 
13962    ins_encode %{
13963      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13964             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13965    %}
13966   ins_pipe(ialu_reg_reg_shift);
13967 %}
13968 
13969 // This pattern is automatically generated from aarch64_ad.m4
13970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13971 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13972 %{
13973   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13974   ins_cost(1.9 * INSN_COST);
13975   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13976 
13977    ins_encode %{
13978      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13979             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13980    %}
13981   ins_pipe(ialu_reg_reg_shift);
13982 %}
13983 
13984 // This pattern is automatically generated from aarch64_ad.m4
13985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13986 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13987 %{
13988   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13989   ins_cost(1.9 * INSN_COST);
13990   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13991 
13992    ins_encode %{
13993      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13994             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13995    %}
13996   ins_pipe(ialu_reg_reg_shift);
13997 %}
13998 
13999 // This pattern is automatically generated from aarch64_ad.m4
14000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14001 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14002 %{
14003   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14004   ins_cost(1.9 * INSN_COST);
14005   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
14006 
14007    ins_encode %{
14008      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14009             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14010    %}
14011   ins_pipe(ialu_reg_reg_shift);
14012 %}
14013 
14014 // This pattern is automatically generated from aarch64_ad.m4
14015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14016 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14017 %{
14018   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14019   ins_cost(1.9 * INSN_COST);
14020   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
14021 
14022    ins_encode %{
14023      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14024             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14025    %}
14026   ins_pipe(ialu_reg_reg_shift);
14027 %}
14028 
14029 // This pattern is automatically generated from aarch64_ad.m4
14030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14031 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14032 %{
14033   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14034   ins_cost(1.9 * INSN_COST);
14035   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
14036 
14037    ins_encode %{
14038      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14039             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14040    %}
14041   ins_pipe(ialu_reg_reg_shift);
14042 %}
14043 
14044 
14045 
14046 // END This section of the file is automatically generated. Do not edit --------------
14047 
14048 
14049 // ============================================================================
14050 // Floating Point Arithmetic Instructions
14051 
14052 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14053   match(Set dst (AddF src1 src2));
14054 
14055   ins_cost(INSN_COST * 5);
14056   format %{ "fadds   $dst, $src1, $src2" %}
14057 
14058   ins_encode %{
14059     __ fadds(as_FloatRegister($dst$$reg),
14060              as_FloatRegister($src1$$reg),
14061              as_FloatRegister($src2$$reg));
14062   %}
14063 
14064   ins_pipe(fp_dop_reg_reg_s);
14065 %}
14066 
14067 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14068   match(Set dst (AddD src1 src2));
14069 
14070   ins_cost(INSN_COST * 5);
14071   format %{ "faddd   $dst, $src1, $src2" %}
14072 
14073   ins_encode %{
14074     __ faddd(as_FloatRegister($dst$$reg),
14075              as_FloatRegister($src1$$reg),
14076              as_FloatRegister($src2$$reg));
14077   %}
14078 
14079   ins_pipe(fp_dop_reg_reg_d);
14080 %}
14081 
14082 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14083   match(Set dst (SubF src1 src2));
14084 
14085   ins_cost(INSN_COST * 5);
14086   format %{ "fsubs   $dst, $src1, $src2" %}
14087 
14088   ins_encode %{
14089     __ fsubs(as_FloatRegister($dst$$reg),
14090              as_FloatRegister($src1$$reg),
14091              as_FloatRegister($src2$$reg));
14092   %}
14093 
14094   ins_pipe(fp_dop_reg_reg_s);
14095 %}
14096 
14097 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14098   match(Set dst (SubD src1 src2));
14099 
14100   ins_cost(INSN_COST * 5);
14101   format %{ "fsubd   $dst, $src1, $src2" %}
14102 
14103   ins_encode %{
14104     __ fsubd(as_FloatRegister($dst$$reg),
14105              as_FloatRegister($src1$$reg),
14106              as_FloatRegister($src2$$reg));
14107   %}
14108 
14109   ins_pipe(fp_dop_reg_reg_d);
14110 %}
14111 
14112 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14113   match(Set dst (MulF src1 src2));
14114 
14115   ins_cost(INSN_COST * 6);
14116   format %{ "fmuls   $dst, $src1, $src2" %}
14117 
14118   ins_encode %{
14119     __ fmuls(as_FloatRegister($dst$$reg),
14120              as_FloatRegister($src1$$reg),
14121              as_FloatRegister($src2$$reg));
14122   %}
14123 
14124   ins_pipe(fp_dop_reg_reg_s);
14125 %}
14126 
14127 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14128   match(Set dst (MulD src1 src2));
14129 
14130   ins_cost(INSN_COST * 6);
14131   format %{ "fmuld   $dst, $src1, $src2" %}
14132 
14133   ins_encode %{
14134     __ fmuld(as_FloatRegister($dst$$reg),
14135              as_FloatRegister($src1$$reg),
14136              as_FloatRegister($src2$$reg));
14137   %}
14138 
14139   ins_pipe(fp_dop_reg_reg_d);
14140 %}
14141 
14142 // src1 * src2 + src3
14143 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14144   predicate(UseFMA);
14145   match(Set dst (FmaF src3 (Binary src1 src2)));
14146 
14147   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14148 
14149   ins_encode %{
14150     __ fmadds(as_FloatRegister($dst$$reg),
14151              as_FloatRegister($src1$$reg),
14152              as_FloatRegister($src2$$reg),
14153              as_FloatRegister($src3$$reg));
14154   %}
14155 
14156   ins_pipe(pipe_class_default);
14157 %}
14158 
14159 // src1 * src2 + src3
14160 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14161   predicate(UseFMA);
14162   match(Set dst (FmaD src3 (Binary src1 src2)));
14163 
14164   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14165 
14166   ins_encode %{
14167     __ fmaddd(as_FloatRegister($dst$$reg),
14168              as_FloatRegister($src1$$reg),
14169              as_FloatRegister($src2$$reg),
14170              as_FloatRegister($src3$$reg));
14171   %}
14172 
14173   ins_pipe(pipe_class_default);
14174 %}
14175 
14176 // -src1 * src2 + src3
14177 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14178   predicate(UseFMA);
14179   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14180   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14181 
14182   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14183 
14184   ins_encode %{
14185     __ fmsubs(as_FloatRegister($dst$$reg),
14186               as_FloatRegister($src1$$reg),
14187               as_FloatRegister($src2$$reg),
14188               as_FloatRegister($src3$$reg));
14189   %}
14190 
14191   ins_pipe(pipe_class_default);
14192 %}
14193 
14194 // -src1 * src2 + src3
14195 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14196   predicate(UseFMA);
14197   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14198   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14199 
14200   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14201 
14202   ins_encode %{
14203     __ fmsubd(as_FloatRegister($dst$$reg),
14204               as_FloatRegister($src1$$reg),
14205               as_FloatRegister($src2$$reg),
14206               as_FloatRegister($src3$$reg));
14207   %}
14208 
14209   ins_pipe(pipe_class_default);
14210 %}
14211 
14212 // -src1 * src2 - src3
14213 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14214   predicate(UseFMA);
14215   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14216   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14217 
14218   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14219 
14220   ins_encode %{
14221     __ fnmadds(as_FloatRegister($dst$$reg),
14222                as_FloatRegister($src1$$reg),
14223                as_FloatRegister($src2$$reg),
14224                as_FloatRegister($src3$$reg));
14225   %}
14226 
14227   ins_pipe(pipe_class_default);
14228 %}
14229 
14230 // -src1 * src2 - src3
14231 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14232   predicate(UseFMA);
14233   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14234   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14235 
14236   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14237 
14238   ins_encode %{
14239     __ fnmaddd(as_FloatRegister($dst$$reg),
14240                as_FloatRegister($src1$$reg),
14241                as_FloatRegister($src2$$reg),
14242                as_FloatRegister($src3$$reg));
14243   %}
14244 
14245   ins_pipe(pipe_class_default);
14246 %}
14247 
14248 // src1 * src2 - src3
14249 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14250   predicate(UseFMA);
14251   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14252 
14253   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14254 
14255   ins_encode %{
14256     __ fnmsubs(as_FloatRegister($dst$$reg),
14257                as_FloatRegister($src1$$reg),
14258                as_FloatRegister($src2$$reg),
14259                as_FloatRegister($src3$$reg));
14260   %}
14261 
14262   ins_pipe(pipe_class_default);
14263 %}
14264 
14265 // src1 * src2 - src3
14266 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14267   predicate(UseFMA);
14268   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14269 
14270   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14271 
14272   ins_encode %{
14273   // n.b. insn name should be fnmsubd
14274     __ fnmsub(as_FloatRegister($dst$$reg),
14275               as_FloatRegister($src1$$reg),
14276               as_FloatRegister($src2$$reg),
14277               as_FloatRegister($src3$$reg));
14278   %}
14279 
14280   ins_pipe(pipe_class_default);
14281 %}
14282 
14283 
14284 // Math.max(FF)F
14285 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14286   match(Set dst (MaxF src1 src2));
14287 
14288   format %{ "fmaxs   $dst, $src1, $src2" %}
14289   ins_encode %{
14290     __ fmaxs(as_FloatRegister($dst$$reg),
14291              as_FloatRegister($src1$$reg),
14292              as_FloatRegister($src2$$reg));
14293   %}
14294 
14295   ins_pipe(fp_dop_reg_reg_s);
14296 %}
14297 
14298 // Math.min(FF)F
14299 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14300   match(Set dst (MinF src1 src2));
14301 
14302   format %{ "fmins   $dst, $src1, $src2" %}
14303   ins_encode %{
14304     __ fmins(as_FloatRegister($dst$$reg),
14305              as_FloatRegister($src1$$reg),
14306              as_FloatRegister($src2$$reg));
14307   %}
14308 
14309   ins_pipe(fp_dop_reg_reg_s);
14310 %}
14311 
14312 // Math.max(DD)D
14313 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14314   match(Set dst (MaxD src1 src2));
14315 
14316   format %{ "fmaxd   $dst, $src1, $src2" %}
14317   ins_encode %{
14318     __ fmaxd(as_FloatRegister($dst$$reg),
14319              as_FloatRegister($src1$$reg),
14320              as_FloatRegister($src2$$reg));
14321   %}
14322 
14323   ins_pipe(fp_dop_reg_reg_d);
14324 %}
14325 
14326 // Math.min(DD)D
14327 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14328   match(Set dst (MinD src1 src2));
14329 
14330   format %{ "fmind   $dst, $src1, $src2" %}
14331   ins_encode %{
14332     __ fmind(as_FloatRegister($dst$$reg),
14333              as_FloatRegister($src1$$reg),
14334              as_FloatRegister($src2$$reg));
14335   %}
14336 
14337   ins_pipe(fp_dop_reg_reg_d);
14338 %}
14339 
14340 
14341 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14342   match(Set dst (DivF src1  src2));
14343 
14344   ins_cost(INSN_COST * 18);
14345   format %{ "fdivs   $dst, $src1, $src2" %}
14346 
14347   ins_encode %{
14348     __ fdivs(as_FloatRegister($dst$$reg),
14349              as_FloatRegister($src1$$reg),
14350              as_FloatRegister($src2$$reg));
14351   %}
14352 
14353   ins_pipe(fp_div_s);
14354 %}
14355 
14356 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14357   match(Set dst (DivD src1  src2));
14358 
14359   ins_cost(INSN_COST * 32);
14360   format %{ "fdivd   $dst, $src1, $src2" %}
14361 
14362   ins_encode %{
14363     __ fdivd(as_FloatRegister($dst$$reg),
14364              as_FloatRegister($src1$$reg),
14365              as_FloatRegister($src2$$reg));
14366   %}
14367 
14368   ins_pipe(fp_div_d);
14369 %}
14370 
14371 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14372   match(Set dst (NegF src));
14373 
14374   ins_cost(INSN_COST * 3);
14375   format %{ "fneg   $dst, $src" %}
14376 
14377   ins_encode %{
14378     __ fnegs(as_FloatRegister($dst$$reg),
14379              as_FloatRegister($src$$reg));
14380   %}
14381 
14382   ins_pipe(fp_uop_s);
14383 %}
14384 
14385 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14386   match(Set dst (NegD src));
14387 
14388   ins_cost(INSN_COST * 3);
14389   format %{ "fnegd   $dst, $src" %}
14390 
14391   ins_encode %{
14392     __ fnegd(as_FloatRegister($dst$$reg),
14393              as_FloatRegister($src$$reg));
14394   %}
14395 
14396   ins_pipe(fp_uop_d);
14397 %}
14398 
14399 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14400 %{
14401   match(Set dst (AbsI src));
14402 
14403   effect(KILL cr);
14404   ins_cost(INSN_COST * 2);
14405   format %{ "cmpw  $src, zr\n\t"
14406             "cnegw $dst, $src, Assembler::LT\t# int abs"
14407   %}
14408 
14409   ins_encode %{
14410     __ cmpw(as_Register($src$$reg), zr);
14411     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14412   %}
14413   ins_pipe(pipe_class_default);
14414 %}
14415 
14416 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14417 %{
14418   match(Set dst (AbsL src));
14419 
14420   effect(KILL cr);
14421   ins_cost(INSN_COST * 2);
14422   format %{ "cmp  $src, zr\n\t"
14423             "cneg $dst, $src, Assembler::LT\t# long abs"
14424   %}
14425 
14426   ins_encode %{
14427     __ cmp(as_Register($src$$reg), zr);
14428     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14429   %}
14430   ins_pipe(pipe_class_default);
14431 %}
14432 
14433 instruct absF_reg(vRegF dst, vRegF src) %{
14434   match(Set dst (AbsF src));
14435 
14436   ins_cost(INSN_COST * 3);
14437   format %{ "fabss   $dst, $src" %}
14438   ins_encode %{
14439     __ fabss(as_FloatRegister($dst$$reg),
14440              as_FloatRegister($src$$reg));
14441   %}
14442 
14443   ins_pipe(fp_uop_s);
14444 %}
14445 
14446 instruct absD_reg(vRegD dst, vRegD src) %{
14447   match(Set dst (AbsD src));
14448 
14449   ins_cost(INSN_COST * 3);
14450   format %{ "fabsd   $dst, $src" %}
14451   ins_encode %{
14452     __ fabsd(as_FloatRegister($dst$$reg),
14453              as_FloatRegister($src$$reg));
14454   %}
14455 
14456   ins_pipe(fp_uop_d);
14457 %}
14458 
14459 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14460   match(Set dst (AbsF (SubF src1 src2)));
14461 
14462   ins_cost(INSN_COST * 3);
14463   format %{ "fabds   $dst, $src1, $src2" %}
14464   ins_encode %{
14465     __ fabds(as_FloatRegister($dst$$reg),
14466              as_FloatRegister($src1$$reg),
14467              as_FloatRegister($src2$$reg));
14468   %}
14469 
14470   ins_pipe(fp_uop_s);
14471 %}
14472 
14473 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14474   match(Set dst (AbsD (SubD src1 src2)));
14475 
14476   ins_cost(INSN_COST * 3);
14477   format %{ "fabdd   $dst, $src1, $src2" %}
14478   ins_encode %{
14479     __ fabdd(as_FloatRegister($dst$$reg),
14480              as_FloatRegister($src1$$reg),
14481              as_FloatRegister($src2$$reg));
14482   %}
14483 
14484   ins_pipe(fp_uop_d);
14485 %}
14486 
14487 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14488   match(Set dst (SqrtD src));
14489 
14490   ins_cost(INSN_COST * 50);
14491   format %{ "fsqrtd  $dst, $src" %}
14492   ins_encode %{
14493     __ fsqrtd(as_FloatRegister($dst$$reg),
14494              as_FloatRegister($src$$reg));
14495   %}
14496 
14497   ins_pipe(fp_div_s);
14498 %}
14499 
14500 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14501   match(Set dst (SqrtF src));
14502 
14503   ins_cost(INSN_COST * 50);
14504   format %{ "fsqrts  $dst, $src" %}
14505   ins_encode %{
14506     __ fsqrts(as_FloatRegister($dst$$reg),
14507              as_FloatRegister($src$$reg));
14508   %}
14509 
14510   ins_pipe(fp_div_d);
14511 %}
14512 
14513 // Math.rint, floor, ceil
14514 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14515   match(Set dst (RoundDoubleMode src rmode));
14516   format %{ "frint  $dst, $src, $rmode" %}
14517   ins_encode %{
14518     switch ($rmode$$constant) {
14519       case RoundDoubleModeNode::rmode_rint:
14520         __ frintnd(as_FloatRegister($dst$$reg),
14521                    as_FloatRegister($src$$reg));
14522         break;
14523       case RoundDoubleModeNode::rmode_floor:
14524         __ frintmd(as_FloatRegister($dst$$reg),
14525                    as_FloatRegister($src$$reg));
14526         break;
14527       case RoundDoubleModeNode::rmode_ceil:
14528         __ frintpd(as_FloatRegister($dst$$reg),
14529                    as_FloatRegister($src$$reg));
14530         break;
14531     }
14532   %}
14533   ins_pipe(fp_uop_d);
14534 %}
14535 
14536 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14537   match(Set dst (CopySignD src1 (Binary src2 zero)));
14538   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14539   format %{ "CopySignD  $dst $src1 $src2" %}
14540   ins_encode %{
14541     FloatRegister dst = as_FloatRegister($dst$$reg),
14542                   src1 = as_FloatRegister($src1$$reg),
14543                   src2 = as_FloatRegister($src2$$reg),
14544                   zero = as_FloatRegister($zero$$reg);
14545     __ fnegd(dst, zero);
14546     __ bsl(dst, __ T8B, src2, src1);
14547   %}
14548   ins_pipe(fp_uop_d);
14549 %}
14550 
14551 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14552   match(Set dst (CopySignF src1 src2));
14553   effect(TEMP_DEF dst, USE src1, USE src2);
14554   format %{ "CopySignF  $dst $src1 $src2" %}
14555   ins_encode %{
14556     FloatRegister dst = as_FloatRegister($dst$$reg),
14557                   src1 = as_FloatRegister($src1$$reg),
14558                   src2 = as_FloatRegister($src2$$reg);
14559     __ movi(dst, __ T2S, 0x80, 24);
14560     __ bsl(dst, __ T8B, src2, src1);
14561   %}
14562   ins_pipe(fp_uop_d);
14563 %}
14564 
14565 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14566   match(Set dst (SignumD src (Binary zero one)));
14567   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14568   format %{ "signumD  $dst, $src" %}
14569   ins_encode %{
14570     FloatRegister src = as_FloatRegister($src$$reg),
14571                   dst = as_FloatRegister($dst$$reg),
14572                   zero = as_FloatRegister($zero$$reg),
14573                   one = as_FloatRegister($one$$reg);
14574     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14575     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14576     // Bit selection instruction gets bit from "one" for each enabled bit in
14577     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14578     // NaN the whole "src" will be copied because "dst" is zero. For all other
14579     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14580     // from "src", and all other bits are copied from 1.0.
14581     __ bsl(dst, __ T8B, one, src);
14582   %}
14583   ins_pipe(fp_uop_d);
14584 %}
14585 
14586 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14587   match(Set dst (SignumF src (Binary zero one)));
14588   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14589   format %{ "signumF  $dst, $src" %}
14590   ins_encode %{
14591     FloatRegister src = as_FloatRegister($src$$reg),
14592                   dst = as_FloatRegister($dst$$reg),
14593                   zero = as_FloatRegister($zero$$reg),
14594                   one = as_FloatRegister($one$$reg);
14595     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14596     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14597     // Bit selection instruction gets bit from "one" for each enabled bit in
14598     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14599     // NaN the whole "src" will be copied because "dst" is zero. For all other
14600     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14601     // from "src", and all other bits are copied from 1.0.
14602     __ bsl(dst, __ T8B, one, src);
14603   %}
14604   ins_pipe(fp_uop_d);
14605 %}
14606 
14607 instruct onspinwait() %{
14608   match(OnSpinWait);
14609   ins_cost(INSN_COST);
14610 
14611   format %{ "onspinwait" %}
14612 
14613   ins_encode %{
14614     __ spin_wait();
14615   %}
14616   ins_pipe(pipe_class_empty);
14617 %}
14618 
14619 // ============================================================================
14620 // Logical Instructions
14621 
14622 // Integer Logical Instructions
14623 
14624 // And Instructions
14625 
14626 
14627 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14628   match(Set dst (AndI src1 src2));
14629 
14630   format %{ "andw  $dst, $src1, $src2\t# int" %}
14631 
14632   ins_cost(INSN_COST);
14633   ins_encode %{
14634     __ andw(as_Register($dst$$reg),
14635             as_Register($src1$$reg),
14636             as_Register($src2$$reg));
14637   %}
14638 
14639   ins_pipe(ialu_reg_reg);
14640 %}
14641 
14642 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14643   match(Set dst (AndI src1 src2));
14644 
14645   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14646 
14647   ins_cost(INSN_COST);
14648   ins_encode %{
14649     __ andw(as_Register($dst$$reg),
14650             as_Register($src1$$reg),
14651             (uint64_t)($src2$$constant));
14652   %}
14653 
14654   ins_pipe(ialu_reg_imm);
14655 %}
14656 
14657 // Or Instructions
14658 
14659 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14660   match(Set dst (OrI src1 src2));
14661 
14662   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14663 
14664   ins_cost(INSN_COST);
14665   ins_encode %{
14666     __ orrw(as_Register($dst$$reg),
14667             as_Register($src1$$reg),
14668             as_Register($src2$$reg));
14669   %}
14670 
14671   ins_pipe(ialu_reg_reg);
14672 %}
14673 
14674 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14675   match(Set dst (OrI src1 src2));
14676 
14677   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14678 
14679   ins_cost(INSN_COST);
14680   ins_encode %{
14681     __ orrw(as_Register($dst$$reg),
14682             as_Register($src1$$reg),
14683             (uint64_t)($src2$$constant));
14684   %}
14685 
14686   ins_pipe(ialu_reg_imm);
14687 %}
14688 
14689 // Xor Instructions
14690 
14691 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14692   match(Set dst (XorI src1 src2));
14693 
14694   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14695 
14696   ins_cost(INSN_COST);
14697   ins_encode %{
14698     __ eorw(as_Register($dst$$reg),
14699             as_Register($src1$$reg),
14700             as_Register($src2$$reg));
14701   %}
14702 
14703   ins_pipe(ialu_reg_reg);
14704 %}
14705 
14706 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14707   match(Set dst (XorI src1 src2));
14708 
14709   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14710 
14711   ins_cost(INSN_COST);
14712   ins_encode %{
14713     __ eorw(as_Register($dst$$reg),
14714             as_Register($src1$$reg),
14715             (uint64_t)($src2$$constant));
14716   %}
14717 
14718   ins_pipe(ialu_reg_imm);
14719 %}
14720 
14721 // Long Logical Instructions
14722 // TODO
14723 
14724 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14725   match(Set dst (AndL src1 src2));
14726 
14727   format %{ "and  $dst, $src1, $src2\t# int" %}
14728 
14729   ins_cost(INSN_COST);
14730   ins_encode %{
14731     __ andr(as_Register($dst$$reg),
14732             as_Register($src1$$reg),
14733             as_Register($src2$$reg));
14734   %}
14735 
14736   ins_pipe(ialu_reg_reg);
14737 %}
14738 
14739 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14740   match(Set dst (AndL src1 src2));
14741 
14742   format %{ "and  $dst, $src1, $src2\t# int" %}
14743 
14744   ins_cost(INSN_COST);
14745   ins_encode %{
14746     __ andr(as_Register($dst$$reg),
14747             as_Register($src1$$reg),
14748             (uint64_t)($src2$$constant));
14749   %}
14750 
14751   ins_pipe(ialu_reg_imm);
14752 %}
14753 
14754 // Or Instructions
14755 
14756 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14757   match(Set dst (OrL src1 src2));
14758 
14759   format %{ "orr  $dst, $src1, $src2\t# int" %}
14760 
14761   ins_cost(INSN_COST);
14762   ins_encode %{
14763     __ orr(as_Register($dst$$reg),
14764            as_Register($src1$$reg),
14765            as_Register($src2$$reg));
14766   %}
14767 
14768   ins_pipe(ialu_reg_reg);
14769 %}
14770 
14771 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14772   match(Set dst (OrL src1 src2));
14773 
14774   format %{ "orr  $dst, $src1, $src2\t# int" %}
14775 
14776   ins_cost(INSN_COST);
14777   ins_encode %{
14778     __ orr(as_Register($dst$$reg),
14779            as_Register($src1$$reg),
14780            (uint64_t)($src2$$constant));
14781   %}
14782 
14783   ins_pipe(ialu_reg_imm);
14784 %}
14785 
14786 // Xor Instructions
14787 
14788 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14789   match(Set dst (XorL src1 src2));
14790 
14791   format %{ "eor  $dst, $src1, $src2\t# int" %}
14792 
14793   ins_cost(INSN_COST);
14794   ins_encode %{
14795     __ eor(as_Register($dst$$reg),
14796            as_Register($src1$$reg),
14797            as_Register($src2$$reg));
14798   %}
14799 
14800   ins_pipe(ialu_reg_reg);
14801 %}
14802 
14803 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14804   match(Set dst (XorL src1 src2));
14805 
14806   ins_cost(INSN_COST);
14807   format %{ "eor  $dst, $src1, $src2\t# int" %}
14808 
14809   ins_encode %{
14810     __ eor(as_Register($dst$$reg),
14811            as_Register($src1$$reg),
14812            (uint64_t)($src2$$constant));
14813   %}
14814 
14815   ins_pipe(ialu_reg_imm);
14816 %}
14817 
14818 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14819 %{
14820   match(Set dst (ConvI2L src));
14821 
14822   ins_cost(INSN_COST);
14823   format %{ "sxtw  $dst, $src\t# i2l" %}
14824   ins_encode %{
14825     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14826   %}
14827   ins_pipe(ialu_reg_shift);
14828 %}
14829 
14830 // this pattern occurs in bigmath arithmetic
14831 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14832 %{
14833   match(Set dst (AndL (ConvI2L src) mask));
14834 
14835   ins_cost(INSN_COST);
14836   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14837   ins_encode %{
14838     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14839   %}
14840 
14841   ins_pipe(ialu_reg_shift);
14842 %}
14843 
14844 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14845   match(Set dst (ConvL2I src));
14846 
14847   ins_cost(INSN_COST);
14848   format %{ "movw  $dst, $src \t// l2i" %}
14849 
14850   ins_encode %{
14851     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14852   %}
14853 
14854   ins_pipe(ialu_reg);
14855 %}
14856 
14857 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14858 %{
14859   match(Set dst (Conv2B src));
14860   effect(KILL cr);
14861 
14862   format %{
14863     "cmpw $src, zr\n\t"
14864     "cset $dst, ne"
14865   %}
14866 
14867   ins_encode %{
14868     __ cmpw(as_Register($src$$reg), zr);
14869     __ cset(as_Register($dst$$reg), Assembler::NE);
14870   %}
14871 
14872   ins_pipe(ialu_reg);
14873 %}
14874 
14875 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14876 %{
14877   match(Set dst (Conv2B src));
14878   effect(KILL cr);
14879 
14880   format %{
14881     "cmp  $src, zr\n\t"
14882     "cset $dst, ne"
14883   %}
14884 
14885   ins_encode %{
14886     __ cmp(as_Register($src$$reg), zr);
14887     __ cset(as_Register($dst$$reg), Assembler::NE);
14888   %}
14889 
14890   ins_pipe(ialu_reg);
14891 %}
14892 
14893 instruct convD2F_reg(vRegF dst, vRegD src) %{
14894   match(Set dst (ConvD2F src));
14895 
14896   ins_cost(INSN_COST * 5);
14897   format %{ "fcvtd  $dst, $src \t// d2f" %}
14898 
14899   ins_encode %{
14900     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14901   %}
14902 
14903   ins_pipe(fp_d2f);
14904 %}
14905 
14906 instruct convF2D_reg(vRegD dst, vRegF src) %{
14907   match(Set dst (ConvF2D src));
14908 
14909   ins_cost(INSN_COST * 5);
14910   format %{ "fcvts  $dst, $src \t// f2d" %}
14911 
14912   ins_encode %{
14913     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14914   %}
14915 
14916   ins_pipe(fp_f2d);
14917 %}
14918 
14919 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14920   match(Set dst (ConvF2I src));
14921 
14922   ins_cost(INSN_COST * 5);
14923   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14924 
14925   ins_encode %{
14926     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14927   %}
14928 
14929   ins_pipe(fp_f2i);
14930 %}
14931 
14932 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14933   match(Set dst (ConvF2L src));
14934 
14935   ins_cost(INSN_COST * 5);
14936   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14937 
14938   ins_encode %{
14939     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14940   %}
14941 
14942   ins_pipe(fp_f2l);
14943 %}
14944 
14945 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14946   match(Set dst (ConvI2F src));
14947 
14948   ins_cost(INSN_COST * 5);
14949   format %{ "scvtfws  $dst, $src \t// i2f" %}
14950 
14951   ins_encode %{
14952     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14953   %}
14954 
14955   ins_pipe(fp_i2f);
14956 %}
14957 
14958 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14959   match(Set dst (ConvL2F src));
14960 
14961   ins_cost(INSN_COST * 5);
14962   format %{ "scvtfs  $dst, $src \t// l2f" %}
14963 
14964   ins_encode %{
14965     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14966   %}
14967 
14968   ins_pipe(fp_l2f);
14969 %}
14970 
14971 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14972   match(Set dst (ConvD2I src));
14973 
14974   ins_cost(INSN_COST * 5);
14975   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14976 
14977   ins_encode %{
14978     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14979   %}
14980 
14981   ins_pipe(fp_d2i);
14982 %}
14983 
14984 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14985   match(Set dst (ConvD2L src));
14986 
14987   ins_cost(INSN_COST * 5);
14988   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14989 
14990   ins_encode %{
14991     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14992   %}
14993 
14994   ins_pipe(fp_d2l);
14995 %}
14996 
14997 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14998   match(Set dst (ConvI2D src));
14999 
15000   ins_cost(INSN_COST * 5);
15001   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15002 
15003   ins_encode %{
15004     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15005   %}
15006 
15007   ins_pipe(fp_i2d);
15008 %}
15009 
15010 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15011   match(Set dst (ConvL2D src));
15012 
15013   ins_cost(INSN_COST * 5);
15014   format %{ "scvtfd  $dst, $src \t// l2d" %}
15015 
15016   ins_encode %{
15017     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15018   %}
15019 
15020   ins_pipe(fp_l2d);
15021 %}
15022 
15023 // stack <-> reg and reg <-> reg shuffles with no conversion
15024 
15025 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15026 
15027   match(Set dst (MoveF2I src));
15028 
15029   effect(DEF dst, USE src);
15030 
15031   ins_cost(4 * INSN_COST);
15032 
15033   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15034 
15035   ins_encode %{
15036     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15037   %}
15038 
15039   ins_pipe(iload_reg_reg);
15040 
15041 %}
15042 
15043 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15044 
15045   match(Set dst (MoveI2F src));
15046 
15047   effect(DEF dst, USE src);
15048 
15049   ins_cost(4 * INSN_COST);
15050 
15051   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15052 
15053   ins_encode %{
15054     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15055   %}
15056 
15057   ins_pipe(pipe_class_memory);
15058 
15059 %}
15060 
15061 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15062 
15063   match(Set dst (MoveD2L src));
15064 
15065   effect(DEF dst, USE src);
15066 
15067   ins_cost(4 * INSN_COST);
15068 
15069   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15070 
15071   ins_encode %{
15072     __ ldr($dst$$Register, Address(sp, $src$$disp));
15073   %}
15074 
15075   ins_pipe(iload_reg_reg);
15076 
15077 %}
15078 
15079 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15080 
15081   match(Set dst (MoveL2D src));
15082 
15083   effect(DEF dst, USE src);
15084 
15085   ins_cost(4 * INSN_COST);
15086 
15087   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15088 
15089   ins_encode %{
15090     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15091   %}
15092 
15093   ins_pipe(pipe_class_memory);
15094 
15095 %}
15096 
15097 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15098 
15099   match(Set dst (MoveF2I src));
15100 
15101   effect(DEF dst, USE src);
15102 
15103   ins_cost(INSN_COST);
15104 
15105   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15106 
15107   ins_encode %{
15108     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15109   %}
15110 
15111   ins_pipe(pipe_class_memory);
15112 
15113 %}
15114 
15115 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15116 
15117   match(Set dst (MoveI2F src));
15118 
15119   effect(DEF dst, USE src);
15120 
15121   ins_cost(INSN_COST);
15122 
15123   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15124 
15125   ins_encode %{
15126     __ strw($src$$Register, Address(sp, $dst$$disp));
15127   %}
15128 
15129   ins_pipe(istore_reg_reg);
15130 
15131 %}
15132 
15133 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15134 
15135   match(Set dst (MoveD2L src));
15136 
15137   effect(DEF dst, USE src);
15138 
15139   ins_cost(INSN_COST);
15140 
15141   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15142 
15143   ins_encode %{
15144     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15145   %}
15146 
15147   ins_pipe(pipe_class_memory);
15148 
15149 %}
15150 
15151 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15152 
15153   match(Set dst (MoveL2D src));
15154 
15155   effect(DEF dst, USE src);
15156 
15157   ins_cost(INSN_COST);
15158 
15159   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15160 
15161   ins_encode %{
15162     __ str($src$$Register, Address(sp, $dst$$disp));
15163   %}
15164 
15165   ins_pipe(istore_reg_reg);
15166 
15167 %}
15168 
15169 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15170 
15171   match(Set dst (MoveF2I src));
15172 
15173   effect(DEF dst, USE src);
15174 
15175   ins_cost(INSN_COST);
15176 
15177   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15178 
15179   ins_encode %{
15180     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15181   %}
15182 
15183   ins_pipe(fp_f2i);
15184 
15185 %}
15186 
15187 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15188 
15189   match(Set dst (MoveI2F src));
15190 
15191   effect(DEF dst, USE src);
15192 
15193   ins_cost(INSN_COST);
15194 
15195   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15196 
15197   ins_encode %{
15198     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15199   %}
15200 
15201   ins_pipe(fp_i2f);
15202 
15203 %}
15204 
15205 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15206 
15207   match(Set dst (MoveD2L src));
15208 
15209   effect(DEF dst, USE src);
15210 
15211   ins_cost(INSN_COST);
15212 
15213   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15214 
15215   ins_encode %{
15216     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15217   %}
15218 
15219   ins_pipe(fp_d2l);
15220 
15221 %}
15222 
15223 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15224 
15225   match(Set dst (MoveL2D src));
15226 
15227   effect(DEF dst, USE src);
15228 
15229   ins_cost(INSN_COST);
15230 
15231   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15232 
15233   ins_encode %{
15234     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15235   %}
15236 
15237   ins_pipe(fp_l2d);
15238 
15239 %}
15240 
15241 // ============================================================================
15242 // clearing of an array
15243 
15244 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15245 %{
15246   match(Set dummy (ClearArray cnt base));
15247   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15248 
15249   ins_cost(4 * INSN_COST);
15250   format %{ "ClearArray $cnt, $base" %}
15251 
15252   ins_encode %{
15253     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15254     if (tpc == NULL) {
15255       ciEnv::current()->record_failure("CodeCache is full");
15256       return;
15257     }
15258   %}
15259 
15260   ins_pipe(pipe_class_memory);
15261 %}
15262 
15263 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15264 %{
15265   predicate((uint64_t)n->in(2)->get_long()
15266             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15267   match(Set dummy (ClearArray cnt base));
15268   effect(TEMP temp, USE_KILL base, KILL cr);
15269 
15270   ins_cost(4 * INSN_COST);
15271   format %{ "ClearArray $cnt, $base" %}
15272 
15273   ins_encode %{
15274     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15275     if (tpc == NULL) {
15276       ciEnv::current()->record_failure("CodeCache is full");
15277       return;
15278     }
15279   %}
15280 
15281   ins_pipe(pipe_class_memory);
15282 %}
15283 
15284 // ============================================================================
15285 // Overflow Math Instructions
15286 
15287 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15288 %{
15289   match(Set cr (OverflowAddI op1 op2));
15290 
15291   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15292   ins_cost(INSN_COST);
15293   ins_encode %{
15294     __ cmnw($op1$$Register, $op2$$Register);
15295   %}
15296 
15297   ins_pipe(icmp_reg_reg);
15298 %}
15299 
15300 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15301 %{
15302   match(Set cr (OverflowAddI op1 op2));
15303 
15304   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15305   ins_cost(INSN_COST);
15306   ins_encode %{
15307     __ cmnw($op1$$Register, $op2$$constant);
15308   %}
15309 
15310   ins_pipe(icmp_reg_imm);
15311 %}
15312 
15313 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15314 %{
15315   match(Set cr (OverflowAddL op1 op2));
15316 
15317   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15318   ins_cost(INSN_COST);
15319   ins_encode %{
15320     __ cmn($op1$$Register, $op2$$Register);
15321   %}
15322 
15323   ins_pipe(icmp_reg_reg);
15324 %}
15325 
15326 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15327 %{
15328   match(Set cr (OverflowAddL op1 op2));
15329 
15330   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15331   ins_cost(INSN_COST);
15332   ins_encode %{
15333     __ adds(zr, $op1$$Register, $op2$$constant);
15334   %}
15335 
15336   ins_pipe(icmp_reg_imm);
15337 %}
15338 
15339 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15340 %{
15341   match(Set cr (OverflowSubI op1 op2));
15342 
15343   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15344   ins_cost(INSN_COST);
15345   ins_encode %{
15346     __ cmpw($op1$$Register, $op2$$Register);
15347   %}
15348 
15349   ins_pipe(icmp_reg_reg);
15350 %}
15351 
15352 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15353 %{
15354   match(Set cr (OverflowSubI op1 op2));
15355 
15356   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15357   ins_cost(INSN_COST);
15358   ins_encode %{
15359     __ cmpw($op1$$Register, $op2$$constant);
15360   %}
15361 
15362   ins_pipe(icmp_reg_imm);
15363 %}
15364 
15365 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15366 %{
15367   match(Set cr (OverflowSubL op1 op2));
15368 
15369   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15370   ins_cost(INSN_COST);
15371   ins_encode %{
15372     __ cmp($op1$$Register, $op2$$Register);
15373   %}
15374 
15375   ins_pipe(icmp_reg_reg);
15376 %}
15377 
15378 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15379 %{
15380   match(Set cr (OverflowSubL op1 op2));
15381 
15382   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15383   ins_cost(INSN_COST);
15384   ins_encode %{
15385     __ subs(zr, $op1$$Register, $op2$$constant);
15386   %}
15387 
15388   ins_pipe(icmp_reg_imm);
15389 %}
15390 
15391 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15392 %{
15393   match(Set cr (OverflowSubI zero op1));
15394 
15395   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15396   ins_cost(INSN_COST);
15397   ins_encode %{
15398     __ cmpw(zr, $op1$$Register);
15399   %}
15400 
15401   ins_pipe(icmp_reg_imm);
15402 %}
15403 
15404 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15405 %{
15406   match(Set cr (OverflowSubL zero op1));
15407 
15408   format %{ "cmp   zr, $op1\t# overflow check long" %}
15409   ins_cost(INSN_COST);
15410   ins_encode %{
15411     __ cmp(zr, $op1$$Register);
15412   %}
15413 
15414   ins_pipe(icmp_reg_imm);
15415 %}
15416 
15417 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15418 %{
15419   match(Set cr (OverflowMulI op1 op2));
15420 
15421   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15422             "cmp   rscratch1, rscratch1, sxtw\n\t"
15423             "movw  rscratch1, #0x80000000\n\t"
15424             "cselw rscratch1, rscratch1, zr, NE\n\t"
15425             "cmpw  rscratch1, #1" %}
15426   ins_cost(5 * INSN_COST);
15427   ins_encode %{
15428     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15429     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15430     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15431     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15432     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15433   %}
15434 
15435   ins_pipe(pipe_slow);
15436 %}
15437 
15438 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15439 %{
15440   match(If cmp (OverflowMulI op1 op2));
15441   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15442             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15443   effect(USE labl, KILL cr);
15444 
15445   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15446             "cmp   rscratch1, rscratch1, sxtw\n\t"
15447             "b$cmp   $labl" %}
15448   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15449   ins_encode %{
15450     Label* L = $labl$$label;
15451     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15452     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15453     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15454     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15455   %}
15456 
15457   ins_pipe(pipe_serial);
15458 %}
15459 
15460 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15461 %{
15462   match(Set cr (OverflowMulL op1 op2));
15463 
15464   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15465             "smulh rscratch2, $op1, $op2\n\t"
15466             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15467             "movw  rscratch1, #0x80000000\n\t"
15468             "cselw rscratch1, rscratch1, zr, NE\n\t"
15469             "cmpw  rscratch1, #1" %}
15470   ins_cost(6 * INSN_COST);
15471   ins_encode %{
15472     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15473     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15474     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15475     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15476     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15477     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15478   %}
15479 
15480   ins_pipe(pipe_slow);
15481 %}
15482 
15483 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15484 %{
15485   match(If cmp (OverflowMulL op1 op2));
15486   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15487             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15488   effect(USE labl, KILL cr);
15489 
15490   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15491             "smulh rscratch2, $op1, $op2\n\t"
15492             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15493             "b$cmp $labl" %}
15494   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15495   ins_encode %{
15496     Label* L = $labl$$label;
15497     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15498     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15499     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15500     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15501     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15502   %}
15503 
15504   ins_pipe(pipe_serial);
15505 %}
15506 
15507 // ============================================================================
15508 // Compare Instructions
15509 
15510 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15511 %{
15512   match(Set cr (CmpI op1 op2));
15513 
15514   effect(DEF cr, USE op1, USE op2);
15515 
15516   ins_cost(INSN_COST);
15517   format %{ "cmpw  $op1, $op2" %}
15518 
15519   ins_encode(aarch64_enc_cmpw(op1, op2));
15520 
15521   ins_pipe(icmp_reg_reg);
15522 %}
15523 
15524 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15525 %{
15526   match(Set cr (CmpI op1 zero));
15527 
15528   effect(DEF cr, USE op1);
15529 
15530   ins_cost(INSN_COST);
15531   format %{ "cmpw $op1, 0" %}
15532 
15533   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15534 
15535   ins_pipe(icmp_reg_imm);
15536 %}
15537 
15538 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15539 %{
15540   match(Set cr (CmpI op1 op2));
15541 
15542   effect(DEF cr, USE op1);
15543 
15544   ins_cost(INSN_COST);
15545   format %{ "cmpw  $op1, $op2" %}
15546 
15547   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15548 
15549   ins_pipe(icmp_reg_imm);
15550 %}
15551 
15552 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15553 %{
15554   match(Set cr (CmpI op1 op2));
15555 
15556   effect(DEF cr, USE op1);
15557 
15558   ins_cost(INSN_COST * 2);
15559   format %{ "cmpw  $op1, $op2" %}
15560 
15561   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15562 
15563   ins_pipe(icmp_reg_imm);
15564 %}
15565 
15566 // Unsigned compare Instructions; really, same as signed compare
15567 // except it should only be used to feed an If or a CMovI which takes a
15568 // cmpOpU.
15569 
15570 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15571 %{
15572   match(Set cr (CmpU op1 op2));
15573 
15574   effect(DEF cr, USE op1, USE op2);
15575 
15576   ins_cost(INSN_COST);
15577   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15578 
15579   ins_encode(aarch64_enc_cmpw(op1, op2));
15580 
15581   ins_pipe(icmp_reg_reg);
15582 %}
15583 
15584 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15585 %{
15586   match(Set cr (CmpU op1 zero));
15587 
15588   effect(DEF cr, USE op1);
15589 
15590   ins_cost(INSN_COST);
15591   format %{ "cmpw $op1, #0\t# unsigned" %}
15592 
15593   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15594 
15595   ins_pipe(icmp_reg_imm);
15596 %}
15597 
15598 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15599 %{
15600   match(Set cr (CmpU op1 op2));
15601 
15602   effect(DEF cr, USE op1);
15603 
15604   ins_cost(INSN_COST);
15605   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15606 
15607   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15608 
15609   ins_pipe(icmp_reg_imm);
15610 %}
15611 
15612 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15613 %{
15614   match(Set cr (CmpU op1 op2));
15615 
15616   effect(DEF cr, USE op1);
15617 
15618   ins_cost(INSN_COST * 2);
15619   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15620 
15621   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15622 
15623   ins_pipe(icmp_reg_imm);
15624 %}
15625 
15626 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15627 %{
15628   match(Set cr (CmpL op1 op2));
15629 
15630   effect(DEF cr, USE op1, USE op2);
15631 
15632   ins_cost(INSN_COST);
15633   format %{ "cmp  $op1, $op2" %}
15634 
15635   ins_encode(aarch64_enc_cmp(op1, op2));
15636 
15637   ins_pipe(icmp_reg_reg);
15638 %}
15639 
15640 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15641 %{
15642   match(Set cr (CmpL op1 zero));
15643 
15644   effect(DEF cr, USE op1);
15645 
15646   ins_cost(INSN_COST);
15647   format %{ "tst  $op1" %}
15648 
15649   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15650 
15651   ins_pipe(icmp_reg_imm);
15652 %}
15653 
15654 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15655 %{
15656   match(Set cr (CmpL op1 op2));
15657 
15658   effect(DEF cr, USE op1);
15659 
15660   ins_cost(INSN_COST);
15661   format %{ "cmp  $op1, $op2" %}
15662 
15663   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15664 
15665   ins_pipe(icmp_reg_imm);
15666 %}
15667 
15668 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15669 %{
15670   match(Set cr (CmpL op1 op2));
15671 
15672   effect(DEF cr, USE op1);
15673 
15674   ins_cost(INSN_COST * 2);
15675   format %{ "cmp  $op1, $op2" %}
15676 
15677   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15678 
15679   ins_pipe(icmp_reg_imm);
15680 %}
15681 
15682 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15683 %{
15684   match(Set cr (CmpUL op1 op2));
15685 
15686   effect(DEF cr, USE op1, USE op2);
15687 
15688   ins_cost(INSN_COST);
15689   format %{ "cmp  $op1, $op2" %}
15690 
15691   ins_encode(aarch64_enc_cmp(op1, op2));
15692 
15693   ins_pipe(icmp_reg_reg);
15694 %}
15695 
15696 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15697 %{
15698   match(Set cr (CmpUL op1 zero));
15699 
15700   effect(DEF cr, USE op1);
15701 
15702   ins_cost(INSN_COST);
15703   format %{ "tst  $op1" %}
15704 
15705   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15706 
15707   ins_pipe(icmp_reg_imm);
15708 %}
15709 
15710 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15711 %{
15712   match(Set cr (CmpUL op1 op2));
15713 
15714   effect(DEF cr, USE op1);
15715 
15716   ins_cost(INSN_COST);
15717   format %{ "cmp  $op1, $op2" %}
15718 
15719   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15720 
15721   ins_pipe(icmp_reg_imm);
15722 %}
15723 
15724 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15725 %{
15726   match(Set cr (CmpUL op1 op2));
15727 
15728   effect(DEF cr, USE op1);
15729 
15730   ins_cost(INSN_COST * 2);
15731   format %{ "cmp  $op1, $op2" %}
15732 
15733   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15734 
15735   ins_pipe(icmp_reg_imm);
15736 %}
15737 
15738 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15739 %{
15740   match(Set cr (CmpP op1 op2));
15741 
15742   effect(DEF cr, USE op1, USE op2);
15743 
15744   ins_cost(INSN_COST);
15745   format %{ "cmp  $op1, $op2\t // ptr" %}
15746 
15747   ins_encode(aarch64_enc_cmpp(op1, op2));
15748 
15749   ins_pipe(icmp_reg_reg);
15750 %}
15751 
15752 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15753 %{
15754   match(Set cr (CmpN op1 op2));
15755 
15756   effect(DEF cr, USE op1, USE op2);
15757 
15758   ins_cost(INSN_COST);
15759   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15760 
15761   ins_encode(aarch64_enc_cmpn(op1, op2));
15762 
15763   ins_pipe(icmp_reg_reg);
15764 %}
15765 
15766 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15767 %{
15768   match(Set cr (CmpP op1 zero));
15769 
15770   effect(DEF cr, USE op1, USE zero);
15771 
15772   ins_cost(INSN_COST);
15773   format %{ "cmp  $op1, 0\t // ptr" %}
15774 
15775   ins_encode(aarch64_enc_testp(op1));
15776 
15777   ins_pipe(icmp_reg_imm);
15778 %}
15779 
15780 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15781 %{
15782   match(Set cr (CmpN op1 zero));
15783 
15784   effect(DEF cr, USE op1, USE zero);
15785 
15786   ins_cost(INSN_COST);
15787   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15788 
15789   ins_encode(aarch64_enc_testn(op1));
15790 
15791   ins_pipe(icmp_reg_imm);
15792 %}
15793 
15794 // FP comparisons
15795 //
15796 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15797 // using normal cmpOp. See declaration of rFlagsReg for details.
15798 
15799 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15800 %{
15801   match(Set cr (CmpF src1 src2));
15802 
15803   ins_cost(3 * INSN_COST);
15804   format %{ "fcmps $src1, $src2" %}
15805 
15806   ins_encode %{
15807     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15808   %}
15809 
15810   ins_pipe(pipe_class_compare);
15811 %}
15812 
15813 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15814 %{
15815   match(Set cr (CmpF src1 src2));
15816 
15817   ins_cost(3 * INSN_COST);
15818   format %{ "fcmps $src1, 0.0" %}
15819 
15820   ins_encode %{
15821     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15822   %}
15823 
15824   ins_pipe(pipe_class_compare);
15825 %}
15826 // FROM HERE
15827 
15828 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15829 %{
15830   match(Set cr (CmpD src1 src2));
15831 
15832   ins_cost(3 * INSN_COST);
15833   format %{ "fcmpd $src1, $src2" %}
15834 
15835   ins_encode %{
15836     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15837   %}
15838 
15839   ins_pipe(pipe_class_compare);
15840 %}
15841 
15842 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15843 %{
15844   match(Set cr (CmpD src1 src2));
15845 
15846   ins_cost(3 * INSN_COST);
15847   format %{ "fcmpd $src1, 0.0" %}
15848 
15849   ins_encode %{
15850     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15851   %}
15852 
15853   ins_pipe(pipe_class_compare);
15854 %}
15855 
15856 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15857 %{
15858   match(Set dst (CmpF3 src1 src2));
15859   effect(KILL cr);
15860 
15861   ins_cost(5 * INSN_COST);
15862   format %{ "fcmps $src1, $src2\n\t"
15863             "csinvw($dst, zr, zr, eq\n\t"
15864             "csnegw($dst, $dst, $dst, lt)"
15865   %}
15866 
15867   ins_encode %{
15868     Label done;
15869     FloatRegister s1 = as_FloatRegister($src1$$reg);
15870     FloatRegister s2 = as_FloatRegister($src2$$reg);
15871     Register d = as_Register($dst$$reg);
15872     __ fcmps(s1, s2);
15873     // installs 0 if EQ else -1
15874     __ csinvw(d, zr, zr, Assembler::EQ);
15875     // keeps -1 if less or unordered else installs 1
15876     __ csnegw(d, d, d, Assembler::LT);
15877     __ bind(done);
15878   %}
15879 
15880   ins_pipe(pipe_class_default);
15881 
15882 %}
15883 
15884 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15885 %{
15886   match(Set dst (CmpD3 src1 src2));
15887   effect(KILL cr);
15888 
15889   ins_cost(5 * INSN_COST);
15890   format %{ "fcmpd $src1, $src2\n\t"
15891             "csinvw($dst, zr, zr, eq\n\t"
15892             "csnegw($dst, $dst, $dst, lt)"
15893   %}
15894 
15895   ins_encode %{
15896     Label done;
15897     FloatRegister s1 = as_FloatRegister($src1$$reg);
15898     FloatRegister s2 = as_FloatRegister($src2$$reg);
15899     Register d = as_Register($dst$$reg);
15900     __ fcmpd(s1, s2);
15901     // installs 0 if EQ else -1
15902     __ csinvw(d, zr, zr, Assembler::EQ);
15903     // keeps -1 if less or unordered else installs 1
15904     __ csnegw(d, d, d, Assembler::LT);
15905     __ bind(done);
15906   %}
15907   ins_pipe(pipe_class_default);
15908 
15909 %}
15910 
15911 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15912 %{
15913   match(Set dst (CmpF3 src1 zero));
15914   effect(KILL cr);
15915 
15916   ins_cost(5 * INSN_COST);
15917   format %{ "fcmps $src1, 0.0\n\t"
15918             "csinvw($dst, zr, zr, eq\n\t"
15919             "csnegw($dst, $dst, $dst, lt)"
15920   %}
15921 
15922   ins_encode %{
15923     Label done;
15924     FloatRegister s1 = as_FloatRegister($src1$$reg);
15925     Register d = as_Register($dst$$reg);
15926     __ fcmps(s1, 0.0);
15927     // installs 0 if EQ else -1
15928     __ csinvw(d, zr, zr, Assembler::EQ);
15929     // keeps -1 if less or unordered else installs 1
15930     __ csnegw(d, d, d, Assembler::LT);
15931     __ bind(done);
15932   %}
15933 
15934   ins_pipe(pipe_class_default);
15935 
15936 %}
15937 
15938 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15939 %{
15940   match(Set dst (CmpD3 src1 zero));
15941   effect(KILL cr);
15942 
15943   ins_cost(5 * INSN_COST);
15944   format %{ "fcmpd $src1, 0.0\n\t"
15945             "csinvw($dst, zr, zr, eq\n\t"
15946             "csnegw($dst, $dst, $dst, lt)"
15947   %}
15948 
15949   ins_encode %{
15950     Label done;
15951     FloatRegister s1 = as_FloatRegister($src1$$reg);
15952     Register d = as_Register($dst$$reg);
15953     __ fcmpd(s1, 0.0);
15954     // installs 0 if EQ else -1
15955     __ csinvw(d, zr, zr, Assembler::EQ);
15956     // keeps -1 if less or unordered else installs 1
15957     __ csnegw(d, d, d, Assembler::LT);
15958     __ bind(done);
15959   %}
15960   ins_pipe(pipe_class_default);
15961 
15962 %}
15963 
15964 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15965 %{
15966   match(Set dst (CmpLTMask p q));
15967   effect(KILL cr);
15968 
15969   ins_cost(3 * INSN_COST);
15970 
15971   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15972             "csetw $dst, lt\n\t"
15973             "subw $dst, zr, $dst"
15974   %}
15975 
15976   ins_encode %{
15977     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15978     __ csetw(as_Register($dst$$reg), Assembler::LT);
15979     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15980   %}
15981 
15982   ins_pipe(ialu_reg_reg);
15983 %}
15984 
15985 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15986 %{
15987   match(Set dst (CmpLTMask src zero));
15988   effect(KILL cr);
15989 
15990   ins_cost(INSN_COST);
15991 
15992   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15993 
15994   ins_encode %{
15995     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15996   %}
15997 
15998   ins_pipe(ialu_reg_shift);
15999 %}
16000 
16001 // ============================================================================
16002 // Max and Min
16003 
16004 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16005 %{
16006   effect( DEF dst, USE src1, USE src2, USE cr );
16007 
16008   ins_cost(INSN_COST * 2);
16009   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
16010 
16011   ins_encode %{
16012     __ cselw(as_Register($dst$$reg),
16013              as_Register($src1$$reg),
16014              as_Register($src2$$reg),
16015              Assembler::LT);
16016   %}
16017 
16018   ins_pipe(icond_reg_reg);
16019 %}
16020 
16021 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16022 %{
16023   match(Set dst (MinI src1 src2));
16024   ins_cost(INSN_COST * 3);
16025 
16026   expand %{
16027     rFlagsReg cr;
16028     compI_reg_reg(cr, src1, src2);
16029     cmovI_reg_reg_lt(dst, src1, src2, cr);
16030   %}
16031 
16032 %}
16033 // FROM HERE
16034 
16035 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16036 %{
16037   effect( DEF dst, USE src1, USE src2, USE cr );
16038 
16039   ins_cost(INSN_COST * 2);
16040   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
16041 
16042   ins_encode %{
16043     __ cselw(as_Register($dst$$reg),
16044              as_Register($src1$$reg),
16045              as_Register($src2$$reg),
16046              Assembler::GT);
16047   %}
16048 
16049   ins_pipe(icond_reg_reg);
16050 %}
16051 
16052 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16053 %{
16054   match(Set dst (MaxI src1 src2));
16055   ins_cost(INSN_COST * 3);
16056   expand %{
16057     rFlagsReg cr;
16058     compI_reg_reg(cr, src1, src2);
16059     cmovI_reg_reg_gt(dst, src1, src2, cr);
16060   %}
16061 %}
16062 
16063 // ============================================================================
16064 // Branch Instructions
16065 
16066 // Direct Branch.
16067 instruct branch(label lbl)
16068 %{
16069   match(Goto);
16070 
16071   effect(USE lbl);
16072 
16073   ins_cost(BRANCH_COST);
16074   format %{ "b  $lbl" %}
16075 
16076   ins_encode(aarch64_enc_b(lbl));
16077 
16078   ins_pipe(pipe_branch);
16079 %}
16080 
16081 // Conditional Near Branch
16082 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16083 %{
16084   // Same match rule as `branchConFar'.
16085   match(If cmp cr);
16086 
16087   effect(USE lbl);
16088 
16089   ins_cost(BRANCH_COST);
16090   // If set to 1 this indicates that the current instruction is a
16091   // short variant of a long branch. This avoids using this
16092   // instruction in first-pass matching. It will then only be used in
16093   // the `Shorten_branches' pass.
16094   // ins_short_branch(1);
16095   format %{ "b$cmp  $lbl" %}
16096 
16097   ins_encode(aarch64_enc_br_con(cmp, lbl));
16098 
16099   ins_pipe(pipe_branch_cond);
16100 %}
16101 
16102 // Conditional Near Branch Unsigned
16103 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16104 %{
16105   // Same match rule as `branchConFar'.
16106   match(If cmp cr);
16107 
16108   effect(USE lbl);
16109 
16110   ins_cost(BRANCH_COST);
16111   // If set to 1 this indicates that the current instruction is a
16112   // short variant of a long branch. This avoids using this
16113   // instruction in first-pass matching. It will then only be used in
16114   // the `Shorten_branches' pass.
16115   // ins_short_branch(1);
16116   format %{ "b$cmp  $lbl\t# unsigned" %}
16117 
16118   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16119 
16120   ins_pipe(pipe_branch_cond);
16121 %}
16122 
16123 // Make use of CBZ and CBNZ.  These instructions, as well as being
16124 // shorter than (cmp; branch), have the additional benefit of not
16125 // killing the flags.
16126 
16127 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16128   match(If cmp (CmpI op1 op2));
16129   effect(USE labl);
16130 
16131   ins_cost(BRANCH_COST);
16132   format %{ "cbw$cmp   $op1, $labl" %}
16133   ins_encode %{
16134     Label* L = $labl$$label;
16135     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16136     if (cond == Assembler::EQ)
16137       __ cbzw($op1$$Register, *L);
16138     else
16139       __ cbnzw($op1$$Register, *L);
16140   %}
16141   ins_pipe(pipe_cmp_branch);
16142 %}
16143 
16144 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16145   match(If cmp (CmpL op1 op2));
16146   effect(USE labl);
16147 
16148   ins_cost(BRANCH_COST);
16149   format %{ "cb$cmp   $op1, $labl" %}
16150   ins_encode %{
16151     Label* L = $labl$$label;
16152     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16153     if (cond == Assembler::EQ)
16154       __ cbz($op1$$Register, *L);
16155     else
16156       __ cbnz($op1$$Register, *L);
16157   %}
16158   ins_pipe(pipe_cmp_branch);
16159 %}
16160 
16161 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16162   match(If cmp (CmpP op1 op2));
16163   effect(USE labl);
16164 
16165   ins_cost(BRANCH_COST);
16166   format %{ "cb$cmp   $op1, $labl" %}
16167   ins_encode %{
16168     Label* L = $labl$$label;
16169     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16170     if (cond == Assembler::EQ)
16171       __ cbz($op1$$Register, *L);
16172     else
16173       __ cbnz($op1$$Register, *L);
16174   %}
16175   ins_pipe(pipe_cmp_branch);
16176 %}
16177 
16178 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16179   match(If cmp (CmpN op1 op2));
16180   effect(USE labl);
16181 
16182   ins_cost(BRANCH_COST);
16183   format %{ "cbw$cmp   $op1, $labl" %}
16184   ins_encode %{
16185     Label* L = $labl$$label;
16186     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16187     if (cond == Assembler::EQ)
16188       __ cbzw($op1$$Register, *L);
16189     else
16190       __ cbnzw($op1$$Register, *L);
16191   %}
16192   ins_pipe(pipe_cmp_branch);
16193 %}
16194 
16195 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16196   match(If cmp (CmpP (DecodeN oop) zero));
16197   effect(USE labl);
16198 
16199   ins_cost(BRANCH_COST);
16200   format %{ "cb$cmp   $oop, $labl" %}
16201   ins_encode %{
16202     Label* L = $labl$$label;
16203     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16204     if (cond == Assembler::EQ)
16205       __ cbzw($oop$$Register, *L);
16206     else
16207       __ cbnzw($oop$$Register, *L);
16208   %}
16209   ins_pipe(pipe_cmp_branch);
16210 %}
16211 
16212 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16213   match(If cmp (CmpU op1 op2));
16214   effect(USE labl);
16215 
16216   ins_cost(BRANCH_COST);
16217   format %{ "cbw$cmp   $op1, $labl" %}
16218   ins_encode %{
16219     Label* L = $labl$$label;
16220     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16221     if (cond == Assembler::EQ || cond == Assembler::LS)
16222       __ cbzw($op1$$Register, *L);
16223     else
16224       __ cbnzw($op1$$Register, *L);
16225   %}
16226   ins_pipe(pipe_cmp_branch);
16227 %}
16228 
16229 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16230   match(If cmp (CmpUL op1 op2));
16231   effect(USE labl);
16232 
16233   ins_cost(BRANCH_COST);
16234   format %{ "cb$cmp   $op1, $labl" %}
16235   ins_encode %{
16236     Label* L = $labl$$label;
16237     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16238     if (cond == Assembler::EQ || cond == Assembler::LS)
16239       __ cbz($op1$$Register, *L);
16240     else
16241       __ cbnz($op1$$Register, *L);
16242   %}
16243   ins_pipe(pipe_cmp_branch);
16244 %}
16245 
16246 // Test bit and Branch
16247 
16248 // Patterns for short (< 32KiB) variants
16249 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16250   match(If cmp (CmpL op1 op2));
16251   effect(USE labl);
16252 
16253   ins_cost(BRANCH_COST);
16254   format %{ "cb$cmp   $op1, $labl # long" %}
16255   ins_encode %{
16256     Label* L = $labl$$label;
16257     Assembler::Condition cond =
16258       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16259     __ tbr(cond, $op1$$Register, 63, *L);
16260   %}
16261   ins_pipe(pipe_cmp_branch);
16262   ins_short_branch(1);
16263 %}
16264 
16265 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16266   match(If cmp (CmpI op1 op2));
16267   effect(USE labl);
16268 
16269   ins_cost(BRANCH_COST);
16270   format %{ "cb$cmp   $op1, $labl # int" %}
16271   ins_encode %{
16272     Label* L = $labl$$label;
16273     Assembler::Condition cond =
16274       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16275     __ tbr(cond, $op1$$Register, 31, *L);
16276   %}
16277   ins_pipe(pipe_cmp_branch);
16278   ins_short_branch(1);
16279 %}
16280 
16281 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16282   match(If cmp (CmpL (AndL op1 op2) op3));
16283   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16284   effect(USE labl);
16285 
16286   ins_cost(BRANCH_COST);
16287   format %{ "tb$cmp   $op1, $op2, $labl" %}
16288   ins_encode %{
16289     Label* L = $labl$$label;
16290     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16291     int bit = exact_log2_long($op2$$constant);
16292     __ tbr(cond, $op1$$Register, bit, *L);
16293   %}
16294   ins_pipe(pipe_cmp_branch);
16295   ins_short_branch(1);
16296 %}
16297 
16298 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16299   match(If cmp (CmpI (AndI op1 op2) op3));
16300   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16301   effect(USE labl);
16302 
16303   ins_cost(BRANCH_COST);
16304   format %{ "tb$cmp   $op1, $op2, $labl" %}
16305   ins_encode %{
16306     Label* L = $labl$$label;
16307     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16308     int bit = exact_log2((juint)$op2$$constant);
16309     __ tbr(cond, $op1$$Register, bit, *L);
16310   %}
16311   ins_pipe(pipe_cmp_branch);
16312   ins_short_branch(1);
16313 %}
16314 
16315 // And far variants
16316 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16317   match(If cmp (CmpL op1 op2));
16318   effect(USE labl);
16319 
16320   ins_cost(BRANCH_COST);
16321   format %{ "cb$cmp   $op1, $labl # long" %}
16322   ins_encode %{
16323     Label* L = $labl$$label;
16324     Assembler::Condition cond =
16325       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16326     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16327   %}
16328   ins_pipe(pipe_cmp_branch);
16329 %}
16330 
16331 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16332   match(If cmp (CmpI op1 op2));
16333   effect(USE labl);
16334 
16335   ins_cost(BRANCH_COST);
16336   format %{ "cb$cmp   $op1, $labl # int" %}
16337   ins_encode %{
16338     Label* L = $labl$$label;
16339     Assembler::Condition cond =
16340       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16341     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16342   %}
16343   ins_pipe(pipe_cmp_branch);
16344 %}
16345 
16346 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16347   match(If cmp (CmpL (AndL op1 op2) op3));
16348   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16349   effect(USE labl);
16350 
16351   ins_cost(BRANCH_COST);
16352   format %{ "tb$cmp   $op1, $op2, $labl" %}
16353   ins_encode %{
16354     Label* L = $labl$$label;
16355     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16356     int bit = exact_log2_long($op2$$constant);
16357     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16358   %}
16359   ins_pipe(pipe_cmp_branch);
16360 %}
16361 
16362 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16363   match(If cmp (CmpI (AndI op1 op2) op3));
16364   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16365   effect(USE labl);
16366 
16367   ins_cost(BRANCH_COST);
16368   format %{ "tb$cmp   $op1, $op2, $labl" %}
16369   ins_encode %{
16370     Label* L = $labl$$label;
16371     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16372     int bit = exact_log2((juint)$op2$$constant);
16373     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16374   %}
16375   ins_pipe(pipe_cmp_branch);
16376 %}
16377 
16378 // Test bits
16379 
16380 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16381   match(Set cr (CmpL (AndL op1 op2) op3));
16382   predicate(Assembler::operand_valid_for_logical_immediate
16383             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16384 
16385   ins_cost(INSN_COST);
16386   format %{ "tst $op1, $op2 # long" %}
16387   ins_encode %{
16388     __ tst($op1$$Register, $op2$$constant);
16389   %}
16390   ins_pipe(ialu_reg_reg);
16391 %}
16392 
16393 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16394   match(Set cr (CmpI (AndI op1 op2) op3));
16395   predicate(Assembler::operand_valid_for_logical_immediate
16396             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16397 
16398   ins_cost(INSN_COST);
16399   format %{ "tst $op1, $op2 # int" %}
16400   ins_encode %{
16401     __ tstw($op1$$Register, $op2$$constant);
16402   %}
16403   ins_pipe(ialu_reg_reg);
16404 %}
16405 
16406 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16407   match(Set cr (CmpL (AndL op1 op2) op3));
16408 
16409   ins_cost(INSN_COST);
16410   format %{ "tst $op1, $op2 # long" %}
16411   ins_encode %{
16412     __ tst($op1$$Register, $op2$$Register);
16413   %}
16414   ins_pipe(ialu_reg_reg);
16415 %}
16416 
16417 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16418   match(Set cr (CmpI (AndI op1 op2) op3));
16419 
16420   ins_cost(INSN_COST);
16421   format %{ "tstw $op1, $op2 # int" %}
16422   ins_encode %{
16423     __ tstw($op1$$Register, $op2$$Register);
16424   %}
16425   ins_pipe(ialu_reg_reg);
16426 %}
16427 
16428 
16429 // Conditional Far Branch
16430 // Conditional Far Branch Unsigned
16431 // TODO: fixme
16432 
16433 // counted loop end branch near
16434 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16435 %{
16436   match(CountedLoopEnd cmp cr);
16437 
16438   effect(USE lbl);
16439 
16440   ins_cost(BRANCH_COST);
16441   // short variant.
16442   // ins_short_branch(1);
16443   format %{ "b$cmp $lbl \t// counted loop end" %}
16444 
16445   ins_encode(aarch64_enc_br_con(cmp, lbl));
16446 
16447   ins_pipe(pipe_branch);
16448 %}
16449 
16450 // counted loop end branch near Unsigned
16451 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16452 %{
16453   match(CountedLoopEnd cmp cr);
16454 
16455   effect(USE lbl);
16456 
16457   ins_cost(BRANCH_COST);
16458   // short variant.
16459   // ins_short_branch(1);
16460   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16461 
16462   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16463 
16464   ins_pipe(pipe_branch);
16465 %}
16466 
16467 // counted loop end branch far
16468 // counted loop end branch far unsigned
16469 // TODO: fixme
16470 
16471 // ============================================================================
16472 // inlined locking and unlocking
16473 
16474 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16475 %{
16476   match(Set cr (FastLock object box));
16477   effect(TEMP tmp, TEMP tmp2);
16478 
16479   // TODO
16480   // identify correct cost
16481   ins_cost(5 * INSN_COST);
16482   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16483 
16484   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16485 
16486   ins_pipe(pipe_serial);
16487 %}
16488 
16489 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16490 %{
16491   match(Set cr (FastUnlock object box));
16492   effect(TEMP tmp, TEMP tmp2);
16493 
16494   ins_cost(5 * INSN_COST);
16495   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16496 
16497   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16498 
16499   ins_pipe(pipe_serial);
16500 %}
16501 
16502 
16503 // ============================================================================
16504 // Safepoint Instructions
16505 
16506 // TODO
16507 // provide a near and far version of this code
16508 
16509 instruct safePoint(rFlagsReg cr, iRegP poll)
16510 %{
16511   match(SafePoint poll);
16512   effect(KILL cr);
16513 
16514   format %{
16515     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16516   %}
16517   ins_encode %{
16518     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16519   %}
16520   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16521 %}
16522 
16523 
16524 // ============================================================================
16525 // Procedure Call/Return Instructions
16526 
16527 // Call Java Static Instruction
16528 
16529 instruct CallStaticJavaDirect(method meth)
16530 %{
16531   match(CallStaticJava);
16532 
16533   effect(USE meth);
16534 
16535   ins_cost(CALL_COST);
16536 
16537   format %{ "call,static $meth \t// ==> " %}
16538 
16539   ins_encode(aarch64_enc_java_static_call(meth),
16540              aarch64_enc_call_epilog);
16541 
16542   ins_pipe(pipe_class_call);
16543 %}
16544 
16545 // TO HERE
16546 
16547 // Call Java Dynamic Instruction
16548 instruct CallDynamicJavaDirect(method meth)
16549 %{
16550   match(CallDynamicJava);
16551 
16552   effect(USE meth);
16553 
16554   ins_cost(CALL_COST);
16555 
16556   format %{ "CALL,dynamic $meth \t// ==> " %}
16557 
16558   ins_encode(aarch64_enc_java_dynamic_call(meth),
16559              aarch64_enc_call_epilog);
16560 
16561   ins_pipe(pipe_class_call);
16562 %}
16563 
16564 // Call Runtime Instruction
16565 
16566 instruct CallRuntimeDirect(method meth)
16567 %{
16568   match(CallRuntime);
16569 
16570   effect(USE meth);
16571 
16572   ins_cost(CALL_COST);
16573 
16574   format %{ "CALL, runtime $meth" %}
16575 
16576   ins_encode( aarch64_enc_java_to_runtime(meth) );
16577 
16578   ins_pipe(pipe_class_call);
16579 %}
16580 
16581 // Call Runtime Instruction
16582 
16583 instruct CallLeafDirect(method meth)
16584 %{
16585   match(CallLeaf);
16586 
16587   effect(USE meth);
16588 
16589   ins_cost(CALL_COST);
16590 
16591   format %{ "CALL, runtime leaf $meth" %}
16592 
16593   ins_encode( aarch64_enc_java_to_runtime(meth) );
16594 
16595   ins_pipe(pipe_class_call);
16596 %}
16597 
16598 // Call Runtime Instruction
16599 
16600 instruct CallLeafNoFPDirect(method meth)
16601 %{
16602   match(CallLeafNoFP);
16603 
16604   effect(USE meth);
16605 
16606   ins_cost(CALL_COST);
16607 
16608   format %{ "CALL, runtime leaf nofp $meth" %}
16609 
16610   ins_encode( aarch64_enc_java_to_runtime(meth) );
16611 
16612   ins_pipe(pipe_class_call);
16613 %}
16614 
16615 instruct CallNativeDirect(method meth)
16616 %{
16617   match(CallNative);
16618 
16619   effect(USE meth);
16620 
16621   ins_cost(CALL_COST);
16622 
16623   format %{ "CALL, native $meth" %}
16624 
16625   ins_encode( aarch64_enc_java_to_runtime(meth) );
16626 
16627   ins_pipe(pipe_class_call);
16628 %}
16629 
16630 // Tail Call; Jump from runtime stub to Java code.
16631 // Also known as an 'interprocedural jump'.
16632 // Target of jump will eventually return to caller.
16633 // TailJump below removes the return address.
16634 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16635 %{
16636   match(TailCall jump_target method_ptr);
16637 
16638   ins_cost(CALL_COST);
16639 
16640   format %{ "br $jump_target\t# $method_ptr holds method" %}
16641 
16642   ins_encode(aarch64_enc_tail_call(jump_target));
16643 
16644   ins_pipe(pipe_class_call);
16645 %}
16646 
16647 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16648 %{
16649   match(TailJump jump_target ex_oop);
16650 
16651   ins_cost(CALL_COST);
16652 
16653   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16654 
16655   ins_encode(aarch64_enc_tail_jmp(jump_target));
16656 
16657   ins_pipe(pipe_class_call);
16658 %}
16659 
16660 // Create exception oop: created by stack-crawling runtime code.
16661 // Created exception is now available to this handler, and is setup
16662 // just prior to jumping to this handler. No code emitted.
16663 // TODO check
16664 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16665 instruct CreateException(iRegP_R0 ex_oop)
16666 %{
16667   match(Set ex_oop (CreateEx));
16668 
16669   format %{ " -- \t// exception oop; no code emitted" %}
16670 
16671   size(0);
16672 
16673   ins_encode( /*empty*/ );
16674 
16675   ins_pipe(pipe_class_empty);
16676 %}
16677 
16678 // Rethrow exception: The exception oop will come in the first
16679 // argument position. Then JUMP (not call) to the rethrow stub code.
16680 instruct RethrowException() %{
16681   match(Rethrow);
16682   ins_cost(CALL_COST);
16683 
16684   format %{ "b rethrow_stub" %}
16685 
16686   ins_encode( aarch64_enc_rethrow() );
16687 
16688   ins_pipe(pipe_class_call);
16689 %}
16690 
16691 
16692 // Return Instruction
16693 // epilog node loads ret address into lr as part of frame pop
16694 instruct Ret()
16695 %{
16696   match(Return);
16697 
16698   format %{ "ret\t// return register" %}
16699 
16700   ins_encode( aarch64_enc_ret() );
16701 
16702   ins_pipe(pipe_branch);
16703 %}
16704 
16705 // Die now.
16706 instruct ShouldNotReachHere() %{
16707   match(Halt);
16708 
16709   ins_cost(CALL_COST);
16710   format %{ "ShouldNotReachHere" %}
16711 
16712   ins_encode %{
16713     if (is_reachable()) {
16714       __ stop(_halt_reason);
16715     }
16716   %}
16717 
16718   ins_pipe(pipe_class_default);
16719 %}
16720 
16721 // ============================================================================
16722 // Partial Subtype Check
16723 //
16724 // superklass array for an instance of the superklass.  Set a hidden
16725 // internal cache on a hit (cache is checked with exposed code in
16726 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16727 // encoding ALSO sets flags.
16728 
16729 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16730 %{
16731   match(Set result (PartialSubtypeCheck sub super));
16732   effect(KILL cr, KILL temp);
16733 
16734   ins_cost(1100);  // slightly larger than the next version
16735   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16736 
16737   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16738 
16739   opcode(0x1); // Force zero of result reg on hit
16740 
16741   ins_pipe(pipe_class_memory);
16742 %}
16743 
16744 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16745 %{
16746   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16747   effect(KILL temp, KILL result);
16748 
16749   ins_cost(1100);  // slightly larger than the next version
16750   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16751 
16752   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16753 
16754   opcode(0x0); // Don't zero result reg on hit
16755 
16756   ins_pipe(pipe_class_memory);
16757 %}
16758 
16759 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16760                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16761 %{
16762   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16763   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16764   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16765 
16766   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16767   ins_encode %{
16768     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16769     __ string_compare($str1$$Register, $str2$$Register,
16770                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16771                       $tmp1$$Register, $tmp2$$Register,
16772                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16773   %}
16774   ins_pipe(pipe_class_memory);
16775 %}
16776 
16777 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16778                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16779 %{
16780   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16781   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16782   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16783 
16784   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16785   ins_encode %{
16786     __ string_compare($str1$$Register, $str2$$Register,
16787                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16788                       $tmp1$$Register, $tmp2$$Register,
16789                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16790   %}
16791   ins_pipe(pipe_class_memory);
16792 %}
16793 
16794 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16795                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16796                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16797 %{
16798   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16799   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16800   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16801          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16802 
16803   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16804   ins_encode %{
16805     __ string_compare($str1$$Register, $str2$$Register,
16806                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16807                       $tmp1$$Register, $tmp2$$Register,
16808                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16809                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16810   %}
16811   ins_pipe(pipe_class_memory);
16812 %}
16813 
16814 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16815                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16816                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16817 %{
16818   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16819   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16820   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16821          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16822 
16823   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16824   ins_encode %{
16825     __ string_compare($str1$$Register, $str2$$Register,
16826                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16827                       $tmp1$$Register, $tmp2$$Register,
16828                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16829                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16830   %}
16831   ins_pipe(pipe_class_memory);
16832 %}
16833 
16834 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16835        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16836        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16837 %{
16838   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16839   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16840   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16841          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16842   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16843 
16844   ins_encode %{
16845     __ string_indexof($str1$$Register, $str2$$Register,
16846                       $cnt1$$Register, $cnt2$$Register,
16847                       $tmp1$$Register, $tmp2$$Register,
16848                       $tmp3$$Register, $tmp4$$Register,
16849                       $tmp5$$Register, $tmp6$$Register,
16850                       -1, $result$$Register, StrIntrinsicNode::UU);
16851   %}
16852   ins_pipe(pipe_class_memory);
16853 %}
16854 
16855 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16856        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16857        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16858 %{
16859   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16860   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16861   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16862          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16863   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16864 
16865   ins_encode %{
16866     __ string_indexof($str1$$Register, $str2$$Register,
16867                       $cnt1$$Register, $cnt2$$Register,
16868                       $tmp1$$Register, $tmp2$$Register,
16869                       $tmp3$$Register, $tmp4$$Register,
16870                       $tmp5$$Register, $tmp6$$Register,
16871                       -1, $result$$Register, StrIntrinsicNode::LL);
16872   %}
16873   ins_pipe(pipe_class_memory);
16874 %}
16875 
16876 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16877        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16878        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16879 %{
16880   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16881   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16882   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16883          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16884   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16885 
16886   ins_encode %{
16887     __ string_indexof($str1$$Register, $str2$$Register,
16888                       $cnt1$$Register, $cnt2$$Register,
16889                       $tmp1$$Register, $tmp2$$Register,
16890                       $tmp3$$Register, $tmp4$$Register,
16891                       $tmp5$$Register, $tmp6$$Register,
16892                       -1, $result$$Register, StrIntrinsicNode::UL);
16893   %}
16894   ins_pipe(pipe_class_memory);
16895 %}
16896 
16897 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16898                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16899                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16900 %{
16901   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16902   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16903   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16904          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16905   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16906 
16907   ins_encode %{
16908     int icnt2 = (int)$int_cnt2$$constant;
16909     __ string_indexof($str1$$Register, $str2$$Register,
16910                       $cnt1$$Register, zr,
16911                       $tmp1$$Register, $tmp2$$Register,
16912                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16913                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16914   %}
16915   ins_pipe(pipe_class_memory);
16916 %}
16917 
16918 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16919                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16920                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16921 %{
16922   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16923   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16924   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16925          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16926   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16927 
16928   ins_encode %{
16929     int icnt2 = (int)$int_cnt2$$constant;
16930     __ string_indexof($str1$$Register, $str2$$Register,
16931                       $cnt1$$Register, zr,
16932                       $tmp1$$Register, $tmp2$$Register,
16933                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16934                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16935   %}
16936   ins_pipe(pipe_class_memory);
16937 %}
16938 
16939 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16940                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16941                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16942 %{
16943   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16944   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16945   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16946          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16947   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16948 
16949   ins_encode %{
16950     int icnt2 = (int)$int_cnt2$$constant;
16951     __ string_indexof($str1$$Register, $str2$$Register,
16952                       $cnt1$$Register, zr,
16953                       $tmp1$$Register, $tmp2$$Register,
16954                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16955                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16956   %}
16957   ins_pipe(pipe_class_memory);
16958 %}
16959 
16960 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16961                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16962                               iRegINoSp tmp3, rFlagsReg cr)
16963 %{
16964   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16965   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16966   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16967          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16968 
16969   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16970 
16971   ins_encode %{
16972     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16973                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16974                            $tmp3$$Register);
16975   %}
16976   ins_pipe(pipe_class_memory);
16977 %}
16978 
16979 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16980                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16981                               iRegINoSp tmp3, rFlagsReg cr)
16982 %{
16983   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16984   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16985   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16986          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16987 
16988   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16989 
16990   ins_encode %{
16991     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16992                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16993                            $tmp3$$Register);
16994   %}
16995   ins_pipe(pipe_class_memory);
16996 %}
16997 
16998 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16999                         iRegI_R0 result, rFlagsReg cr)
17000 %{
17001   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17002   match(Set result (StrEquals (Binary str1 str2) cnt));
17003   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17004 
17005   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17006   ins_encode %{
17007     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17008     __ string_equals($str1$$Register, $str2$$Register,
17009                      $result$$Register, $cnt$$Register, 1);
17010   %}
17011   ins_pipe(pipe_class_memory);
17012 %}
17013 
17014 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17015                         iRegI_R0 result, rFlagsReg cr)
17016 %{
17017   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17018   match(Set result (StrEquals (Binary str1 str2) cnt));
17019   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17020 
17021   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17022   ins_encode %{
17023     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17024     __ string_equals($str1$$Register, $str2$$Register,
17025                      $result$$Register, $cnt$$Register, 2);
17026   %}
17027   ins_pipe(pipe_class_memory);
17028 %}
17029 
17030 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17031                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17032                        iRegP_R10 tmp, rFlagsReg cr)
17033 %{
17034   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17035   match(Set result (AryEq ary1 ary2));
17036   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17037 
17038   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17039   ins_encode %{
17040     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17041                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17042                                    $result$$Register, $tmp$$Register, 1);
17043     if (tpc == NULL) {
17044       ciEnv::current()->record_failure("CodeCache is full");
17045       return;
17046     }
17047   %}
17048   ins_pipe(pipe_class_memory);
17049 %}
17050 
17051 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17052                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17053                        iRegP_R10 tmp, rFlagsReg cr)
17054 %{
17055   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17056   match(Set result (AryEq ary1 ary2));
17057   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17058 
17059   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17060   ins_encode %{
17061     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17062                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17063                                    $result$$Register, $tmp$$Register, 2);
17064     if (tpc == NULL) {
17065       ciEnv::current()->record_failure("CodeCache is full");
17066       return;
17067     }
17068   %}
17069   ins_pipe(pipe_class_memory);
17070 %}
17071 
17072 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17073 %{
17074   match(Set result (HasNegatives ary1 len));
17075   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17076   format %{ "has negatives byte[] $ary1,$len -> $result" %}
17077   ins_encode %{
17078     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17079     if (tpc == NULL) {
17080       ciEnv::current()->record_failure("CodeCache is full");
17081       return;
17082     }
17083   %}
17084   ins_pipe( pipe_slow );
17085 %}
17086 
17087 // fast char[] to byte[] compression
17088 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17089                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17090                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17091                          iRegI_R0 result, rFlagsReg cr)
17092 %{
17093   match(Set result (StrCompressedCopy src (Binary dst len)));
17094   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17095 
17096   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
17097   ins_encode %{
17098     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17099                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17100                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17101                            $result$$Register);
17102   %}
17103   ins_pipe( pipe_slow );
17104 %}
17105 
17106 // fast byte[] to char[] inflation
17107 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17108                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17109 %{
17110   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17111   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17112 
17113   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17114   ins_encode %{
17115     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17116                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17117                                         $tmp3$$FloatRegister, $tmp4$$Register);
17118     if (tpc == NULL) {
17119       ciEnv::current()->record_failure("CodeCache is full");
17120       return;
17121     }
17122   %}
17123   ins_pipe(pipe_class_memory);
17124 %}
17125 
17126 // encode char[] to byte[] in ISO_8859_1
17127 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17128                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17129                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17130                           iRegI_R0 result, rFlagsReg cr)
17131 %{
17132   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17133   match(Set result (EncodeISOArray src (Binary dst len)));
17134   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17135          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17136 
17137   format %{ "Encode array $src,$dst,$len -> $result" %}
17138   ins_encode %{
17139     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17140          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
17141          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
17142   %}
17143   ins_pipe( pipe_class_memory );
17144 %}
17145 
17146 // ============================================================================
17147 // This name is KNOWN by the ADLC and cannot be changed.
17148 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17149 // for this guy.
17150 instruct tlsLoadP(thread_RegP dst)
17151 %{
17152   match(Set dst (ThreadLocal));
17153 
17154   ins_cost(0);
17155 
17156   format %{ " -- \t// $dst=Thread::current(), empty" %}
17157 
17158   size(0);
17159 
17160   ins_encode( /*empty*/ );
17161 
17162   ins_pipe(pipe_class_empty);
17163 %}
17164 
17165 //----------PEEPHOLE RULES-----------------------------------------------------
17166 // These must follow all instruction definitions as they use the names
17167 // defined in the instructions definitions.
17168 //
17169 // peepmatch ( root_instr_name [preceding_instruction]* );
17170 //
17171 // peepconstraint %{
17172 // (instruction_number.operand_name relational_op instruction_number.operand_name
17173 //  [, ...] );
17174 // // instruction numbers are zero-based using left to right order in peepmatch
17175 //
17176 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17177 // // provide an instruction_number.operand_name for each operand that appears
17178 // // in the replacement instruction's match rule
17179 //
17180 // ---------VM FLAGS---------------------------------------------------------
17181 //
17182 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17183 //
17184 // Each peephole rule is given an identifying number starting with zero and
17185 // increasing by one in the order seen by the parser.  An individual peephole
17186 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17187 // on the command-line.
17188 //
17189 // ---------CURRENT LIMITATIONS----------------------------------------------
17190 //
17191 // Only match adjacent instructions in same basic block
17192 // Only equality constraints
17193 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17194 // Only one replacement instruction
17195 //
17196 // ---------EXAMPLE----------------------------------------------------------
17197 //
17198 // // pertinent parts of existing instructions in architecture description
17199 // instruct movI(iRegINoSp dst, iRegI src)
17200 // %{
17201 //   match(Set dst (CopyI src));
17202 // %}
17203 //
17204 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17205 // %{
17206 //   match(Set dst (AddI dst src));
17207 //   effect(KILL cr);
17208 // %}
17209 //
17210 // // Change (inc mov) to lea
17211 // peephole %{
17212 //   // increment preceeded by register-register move
17213 //   peepmatch ( incI_iReg movI );
17214 //   // require that the destination register of the increment
17215 //   // match the destination register of the move
17216 //   peepconstraint ( 0.dst == 1.dst );
17217 //   // construct a replacement instruction that sets
17218 //   // the destination to ( move's source register + one )
17219 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17220 // %}
17221 //
17222 
17223 // Implementation no longer uses movX instructions since
17224 // machine-independent system no longer uses CopyX nodes.
17225 //
17226 // peephole
17227 // %{
17228 //   peepmatch (incI_iReg movI);
17229 //   peepconstraint (0.dst == 1.dst);
17230 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17231 // %}
17232 
17233 // peephole
17234 // %{
17235 //   peepmatch (decI_iReg movI);
17236 //   peepconstraint (0.dst == 1.dst);
17237 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17238 // %}
17239 
17240 // peephole
17241 // %{
17242 //   peepmatch (addI_iReg_imm movI);
17243 //   peepconstraint (0.dst == 1.dst);
17244 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17245 // %}
17246 
17247 // peephole
17248 // %{
17249 //   peepmatch (incL_iReg movL);
17250 //   peepconstraint (0.dst == 1.dst);
17251 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17252 // %}
17253 
17254 // peephole
17255 // %{
17256 //   peepmatch (decL_iReg movL);
17257 //   peepconstraint (0.dst == 1.dst);
17258 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17259 // %}
17260 
17261 // peephole
17262 // %{
17263 //   peepmatch (addL_iReg_imm movL);
17264 //   peepconstraint (0.dst == 1.dst);
17265 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17266 // %}
17267 
17268 // peephole
17269 // %{
17270 //   peepmatch (addP_iReg_imm movP);
17271 //   peepconstraint (0.dst == 1.dst);
17272 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17273 // %}
17274 
17275 // // Change load of spilled value to only a spill
17276 // instruct storeI(memory mem, iRegI src)
17277 // %{
17278 //   match(Set mem (StoreI mem src));
17279 // %}
17280 //
17281 // instruct loadI(iRegINoSp dst, memory mem)
17282 // %{
17283 //   match(Set dst (LoadI mem));
17284 // %}
17285 //
17286 
17287 //----------SMARTSPILL RULES---------------------------------------------------
17288 // These must follow all instruction definitions as they use the names
17289 // defined in the instructions definitions.
17290 
17291 // Local Variables:
17292 // mode: c++
17293 // End: