1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 reg_class p0_reg(P0);
 1203 reg_class p1_reg(P1);
 1204 
 1205 // Singleton class for condition codes
 1206 reg_class int_flags(RFLAGS);
 1207 
 1208 %}
 1209 
 1210 //----------DEFINITION BLOCK---------------------------------------------------
 1211 // Define name --> value mappings to inform the ADLC of an integer valued name
 1212 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1213 // Format:
 1214 //        int_def  <name>         ( <int_value>, <expression>);
 1215 // Generated Code in ad_<arch>.hpp
 1216 //        #define  <name>   (<expression>)
 1217 //        // value == <int_value>
 1218 // Generated code in ad_<arch>.cpp adlc_verification()
 1219 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1220 //
 1221 
 1222 // we follow the ppc-aix port in using a simple cost model which ranks
 1223 // register operations as cheap, memory ops as more expensive and
 1224 // branches as most expensive. the first two have a low as well as a
 1225 // normal cost. huge cost appears to be a way of saying don't do
 1226 // something
 1227 
 1228 definitions %{
 1229   // The default cost (of a register move instruction).
 1230   int_def INSN_COST            (    100,     100);
 1231   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1232   int_def CALL_COST            (    200,     2 * INSN_COST);
 1233   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1234 %}
 1235 
 1236 
 1237 //----------SOURCE BLOCK-------------------------------------------------------
 1238 // This is a block of C++ code which provides values, functions, and
 1239 // definitions necessary in the rest of the architecture description
 1240 
 1241 source_hpp %{
 1242 
 1243 #include "asm/macroAssembler.hpp"
 1244 #include "gc/shared/barrierSetAssembler.hpp"
 1245 #include "gc/shared/cardTable.hpp"
 1246 #include "gc/shared/cardTableBarrierSet.hpp"
 1247 #include "gc/shared/collectedHeap.hpp"
 1248 #include "opto/addnode.hpp"
 1249 #include "opto/convertnode.hpp"
 1250 #include "runtime/objectMonitor.hpp"
 1251 
 1252 extern RegMask _ANY_REG32_mask;
 1253 extern RegMask _ANY_REG_mask;
 1254 extern RegMask _PTR_REG_mask;
 1255 extern RegMask _NO_SPECIAL_REG32_mask;
 1256 extern RegMask _NO_SPECIAL_REG_mask;
 1257 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1258 
 1259 class CallStubImpl {
 1260 
 1261   //--------------------------------------------------------------
 1262   //---<  Used for optimization in Compile::shorten_branches  >---
 1263   //--------------------------------------------------------------
 1264 
 1265  public:
 1266   // Size of call trampoline stub.
 1267   static uint size_call_trampoline() {
 1268     return 0; // no call trampolines on this platform
 1269   }
 1270 
 1271   // number of relocations needed by a call trampoline stub
 1272   static uint reloc_call_trampoline() {
 1273     return 0; // no call trampolines on this platform
 1274   }
 1275 };
 1276 
 1277 class HandlerImpl {
 1278 
 1279  public:
 1280 
 1281   static int emit_exception_handler(CodeBuffer &cbuf);
 1282   static int emit_deopt_handler(CodeBuffer& cbuf);
 1283 
 1284   static uint size_exception_handler() {
 1285     return MacroAssembler::far_branch_size();
 1286   }
 1287 
 1288   static uint size_deopt_handler() {
 1289     // count one adr and one far branch instruction
 1290     return 4 * NativeInstruction::instruction_size;
 1291   }
 1292 };
 1293 
 1294 class Node::PD {
 1295 public:
 1296   enum NodeFlags {
 1297     _last_flag = Node::_last_flag
 1298   };
 1299 };
 1300 
 1301   bool is_CAS(int opcode, bool maybe_volatile);
 1302 
 1303   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1304 
 1305   bool unnecessary_acquire(const Node *barrier);
 1306   bool needs_acquiring_load(const Node *load);
 1307 
 1308   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1309 
 1310   bool unnecessary_release(const Node *barrier);
 1311   bool unnecessary_volatile(const Node *barrier);
 1312   bool needs_releasing_store(const Node *store);
 1313 
 1314   // predicate controlling translation of CompareAndSwapX
 1315   bool needs_acquiring_load_exclusive(const Node *load);
 1316 
 1317   // predicate controlling addressing modes
 1318   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1319 %}
 1320 
 1321 source %{
 1322 
 1323   // Derived RegMask with conditionally allocatable registers
 1324 
 1325   void PhaseOutput::pd_perform_mach_node_analysis() {
 1326   }
 1327 
 1328   int MachNode::pd_alignment_required() const {
 1329     return 1;
 1330   }
 1331 
 1332   int MachNode::compute_padding(int current_offset) const {
 1333     return 0;
 1334   }
 1335 
 1336   RegMask _ANY_REG32_mask;
 1337   RegMask _ANY_REG_mask;
 1338   RegMask _PTR_REG_mask;
 1339   RegMask _NO_SPECIAL_REG32_mask;
 1340   RegMask _NO_SPECIAL_REG_mask;
 1341   RegMask _NO_SPECIAL_PTR_REG_mask;
 1342 
 1343   void reg_mask_init() {
 1344     // We derive below RegMask(s) from the ones which are auto-generated from
 1345     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1346     // registers conditionally reserved.
 1347 
 1348     _ANY_REG32_mask = _ALL_REG32_mask;
 1349     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1350 
 1351     _ANY_REG_mask = _ALL_REG_mask;
 1352 
 1353     _PTR_REG_mask = _ALL_REG_mask;
 1354 
 1355     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1356     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1357 
 1358     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1362     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1363 
 1364     // r27 is not allocatable when compressed oops is on and heapbase is not
 1365     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1366     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1367       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1368       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1369       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1370     }
 1371 
 1372     // r29 is not allocatable when PreserveFramePointer is on
 1373     if (PreserveFramePointer) {
 1374       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1375       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1376       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1377     }
 1378   }
 1379 
 1380   // Optimizaton of volatile gets and puts
 1381   // -------------------------------------
 1382   //
 1383   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1384   // use to implement volatile reads and writes. For a volatile read
 1385   // we simply need
 1386   //
 1387   //   ldar<x>
 1388   //
 1389   // and for a volatile write we need
 1390   //
 1391   //   stlr<x>
 1392   //
 1393   // Alternatively, we can implement them by pairing a normal
 1394   // load/store with a memory barrier. For a volatile read we need
 1395   //
 1396   //   ldr<x>
 1397   //   dmb ishld
 1398   //
 1399   // for a volatile write
 1400   //
 1401   //   dmb ish
 1402   //   str<x>
 1403   //   dmb ish
 1404   //
 1405   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1406   // sequences. These are normally translated to an instruction
 1407   // sequence like the following
 1408   //
 1409   //   dmb      ish
 1410   // retry:
 1411   //   ldxr<x>   rval raddr
 1412   //   cmp       rval rold
 1413   //   b.ne done
 1414   //   stlxr<x>  rval, rnew, rold
 1415   //   cbnz      rval retry
 1416   // done:
 1417   //   cset      r0, eq
 1418   //   dmb ishld
 1419   //
 1420   // Note that the exclusive store is already using an stlxr
 1421   // instruction. That is required to ensure visibility to other
 1422   // threads of the exclusive write (assuming it succeeds) before that
 1423   // of any subsequent writes.
 1424   //
 1425   // The following instruction sequence is an improvement on the above
 1426   //
 1427   // retry:
 1428   //   ldaxr<x>  rval raddr
 1429   //   cmp       rval rold
 1430   //   b.ne done
 1431   //   stlxr<x>  rval, rnew, rold
 1432   //   cbnz      rval retry
 1433   // done:
 1434   //   cset      r0, eq
 1435   //
 1436   // We don't need the leading dmb ish since the stlxr guarantees
 1437   // visibility of prior writes in the case that the swap is
 1438   // successful. Crucially we don't have to worry about the case where
 1439   // the swap is not successful since no valid program should be
 1440   // relying on visibility of prior changes by the attempting thread
 1441   // in the case where the CAS fails.
 1442   //
 1443   // Similarly, we don't need the trailing dmb ishld if we substitute
 1444   // an ldaxr instruction since that will provide all the guarantees we
 1445   // require regarding observation of changes made by other threads
 1446   // before any change to the CAS address observed by the load.
 1447   //
 1448   // In order to generate the desired instruction sequence we need to
 1449   // be able to identify specific 'signature' ideal graph node
 1450   // sequences which i) occur as a translation of a volatile reads or
 1451   // writes or CAS operations and ii) do not occur through any other
 1452   // translation or graph transformation. We can then provide
 1453   // alternative aldc matching rules which translate these node
 1454   // sequences to the desired machine code sequences. Selection of the
 1455   // alternative rules can be implemented by predicates which identify
 1456   // the relevant node sequences.
 1457   //
 1458   // The ideal graph generator translates a volatile read to the node
 1459   // sequence
 1460   //
 1461   //   LoadX[mo_acquire]
 1462   //   MemBarAcquire
 1463   //
 1464   // As a special case when using the compressed oops optimization we
 1465   // may also see this variant
 1466   //
 1467   //   LoadN[mo_acquire]
 1468   //   DecodeN
 1469   //   MemBarAcquire
 1470   //
 1471   // A volatile write is translated to the node sequence
 1472   //
 1473   //   MemBarRelease
 1474   //   StoreX[mo_release] {CardMark}-optional
 1475   //   MemBarVolatile
 1476   //
 1477   // n.b. the above node patterns are generated with a strict
 1478   // 'signature' configuration of input and output dependencies (see
 1479   // the predicates below for exact details). The card mark may be as
 1480   // simple as a few extra nodes or, in a few GC configurations, may
 1481   // include more complex control flow between the leading and
 1482   // trailing memory barriers. However, whatever the card mark
 1483   // configuration these signatures are unique to translated volatile
 1484   // reads/stores -- they will not appear as a result of any other
 1485   // bytecode translation or inlining nor as a consequence of
 1486   // optimizing transforms.
 1487   //
 1488   // We also want to catch inlined unsafe volatile gets and puts and
 1489   // be able to implement them using either ldar<x>/stlr<x> or some
 1490   // combination of ldr<x>/stlr<x> and dmb instructions.
 1491   //
 1492   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1493   // normal volatile put node sequence containing an extra cpuorder
 1494   // membar
 1495   //
 1496   //   MemBarRelease
 1497   //   MemBarCPUOrder
 1498   //   StoreX[mo_release] {CardMark}-optional
 1499   //   MemBarCPUOrder
 1500   //   MemBarVolatile
 1501   //
 1502   // n.b. as an aside, a cpuorder membar is not itself subject to
 1503   // matching and translation by adlc rules.  However, the rule
 1504   // predicates need to detect its presence in order to correctly
 1505   // select the desired adlc rules.
 1506   //
 1507   // Inlined unsafe volatile gets manifest as a slightly different
 1508   // node sequence to a normal volatile get because of the
 1509   // introduction of some CPUOrder memory barriers to bracket the
 1510   // Load. However, but the same basic skeleton of a LoadX feeding a
 1511   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1512   // present
 1513   //
 1514   //   MemBarCPUOrder
 1515   //        ||       \\
 1516   //   MemBarCPUOrder LoadX[mo_acquire]
 1517   //        ||            |
 1518   //        ||       {DecodeN} optional
 1519   //        ||       /
 1520   //     MemBarAcquire
 1521   //
 1522   // In this case the acquire membar does not directly depend on the
 1523   // load. However, we can be sure that the load is generated from an
 1524   // inlined unsafe volatile get if we see it dependent on this unique
 1525   // sequence of membar nodes. Similarly, given an acquire membar we
 1526   // can know that it was added because of an inlined unsafe volatile
 1527   // get if it is fed and feeds a cpuorder membar and if its feed
 1528   // membar also feeds an acquiring load.
 1529   //
 1530   // Finally an inlined (Unsafe) CAS operation is translated to the
 1531   // following ideal graph
 1532   //
 1533   //   MemBarRelease
 1534   //   MemBarCPUOrder
 1535   //   CompareAndSwapX {CardMark}-optional
 1536   //   MemBarCPUOrder
 1537   //   MemBarAcquire
 1538   //
 1539   // So, where we can identify these volatile read and write
 1540   // signatures we can choose to plant either of the above two code
 1541   // sequences. For a volatile read we can simply plant a normal
 1542   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1543   // also choose to inhibit translation of the MemBarAcquire and
 1544   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1545   //
 1546   // When we recognise a volatile store signature we can choose to
 1547   // plant at a dmb ish as a translation for the MemBarRelease, a
 1548   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1549   // Alternatively, we can inhibit translation of the MemBarRelease
 1550   // and MemBarVolatile and instead plant a simple stlr<x>
 1551   // instruction.
 1552   //
 1553   // when we recognise a CAS signature we can choose to plant a dmb
 1554   // ish as a translation for the MemBarRelease, the conventional
 1555   // macro-instruction sequence for the CompareAndSwap node (which
 1556   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1557   // Alternatively, we can elide generation of the dmb instructions
 1558   // and plant the alternative CompareAndSwap macro-instruction
 1559   // sequence (which uses ldaxr<x>).
 1560   //
 1561   // Of course, the above only applies when we see these signature
 1562   // configurations. We still want to plant dmb instructions in any
 1563   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1564   // MemBarVolatile. For example, at the end of a constructor which
 1565   // writes final/volatile fields we will see a MemBarRelease
 1566   // instruction and this needs a 'dmb ish' lest we risk the
 1567   // constructed object being visible without making the
 1568   // final/volatile field writes visible.
 1569   //
 1570   // n.b. the translation rules below which rely on detection of the
 1571   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1572   // If we see anything other than the signature configurations we
 1573   // always just translate the loads and stores to ldr<x> and str<x>
 1574   // and translate acquire, release and volatile membars to the
 1575   // relevant dmb instructions.
 1576   //
 1577 
 1578   // is_CAS(int opcode, bool maybe_volatile)
 1579   //
 1580   // return true if opcode is one of the possible CompareAndSwapX
 1581   // values otherwise false.
 1582 
 1583   bool is_CAS(int opcode, bool maybe_volatile)
 1584   {
 1585     switch(opcode) {
 1586       // We handle these
 1587     case Op_CompareAndSwapI:
 1588     case Op_CompareAndSwapL:
 1589     case Op_CompareAndSwapP:
 1590     case Op_CompareAndSwapN:
 1591     case Op_ShenandoahCompareAndSwapP:
 1592     case Op_ShenandoahCompareAndSwapN:
 1593     case Op_CompareAndSwapB:
 1594     case Op_CompareAndSwapS:
 1595     case Op_GetAndSetI:
 1596     case Op_GetAndSetL:
 1597     case Op_GetAndSetP:
 1598     case Op_GetAndSetN:
 1599     case Op_GetAndAddI:
 1600     case Op_GetAndAddL:
 1601       return true;
 1602     case Op_CompareAndExchangeI:
 1603     case Op_CompareAndExchangeN:
 1604     case Op_CompareAndExchangeB:
 1605     case Op_CompareAndExchangeS:
 1606     case Op_CompareAndExchangeL:
 1607     case Op_CompareAndExchangeP:
 1608     case Op_WeakCompareAndSwapB:
 1609     case Op_WeakCompareAndSwapS:
 1610     case Op_WeakCompareAndSwapI:
 1611     case Op_WeakCompareAndSwapL:
 1612     case Op_WeakCompareAndSwapP:
 1613     case Op_WeakCompareAndSwapN:
 1614     case Op_ShenandoahWeakCompareAndSwapP:
 1615     case Op_ShenandoahWeakCompareAndSwapN:
 1616     case Op_ShenandoahCompareAndExchangeP:
 1617     case Op_ShenandoahCompareAndExchangeN:
 1618       return maybe_volatile;
 1619     default:
 1620       return false;
 1621     }
 1622   }
 1623 
 1624   // helper to determine the maximum number of Phi nodes we may need to
 1625   // traverse when searching from a card mark membar for the merge mem
 1626   // feeding a trailing membar or vice versa
 1627 
 1628 // predicates controlling emit of ldr<x>/ldar<x>
 1629 
 1630 bool unnecessary_acquire(const Node *barrier)
 1631 {
 1632   assert(barrier->is_MemBar(), "expecting a membar");
 1633 
 1634   MemBarNode* mb = barrier->as_MemBar();
 1635 
 1636   if (mb->trailing_load()) {
 1637     return true;
 1638   }
 1639 
 1640   if (mb->trailing_load_store()) {
 1641     Node* load_store = mb->in(MemBarNode::Precedent);
 1642     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1643     return is_CAS(load_store->Opcode(), true);
 1644   }
 1645 
 1646   return false;
 1647 }
 1648 
 1649 bool needs_acquiring_load(const Node *n)
 1650 {
 1651   assert(n->is_Load(), "expecting a load");
 1652   LoadNode *ld = n->as_Load();
 1653   return ld->is_acquire();
 1654 }
 1655 
 1656 bool unnecessary_release(const Node *n)
 1657 {
 1658   assert((n->is_MemBar() &&
 1659           n->Opcode() == Op_MemBarRelease),
 1660          "expecting a release membar");
 1661 
 1662   MemBarNode *barrier = n->as_MemBar();
 1663   if (!barrier->leading()) {
 1664     return false;
 1665   } else {
 1666     Node* trailing = barrier->trailing_membar();
 1667     MemBarNode* trailing_mb = trailing->as_MemBar();
 1668     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1669     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1670 
 1671     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1672     if (mem->is_Store()) {
 1673       assert(mem->as_Store()->is_release(), "");
 1674       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1675       return true;
 1676     } else {
 1677       assert(mem->is_LoadStore(), "");
 1678       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1679       return is_CAS(mem->Opcode(), true);
 1680     }
 1681   }
 1682   return false;
 1683 }
 1684 
 1685 bool unnecessary_volatile(const Node *n)
 1686 {
 1687   // assert n->is_MemBar();
 1688   MemBarNode *mbvol = n->as_MemBar();
 1689 
 1690   bool release = mbvol->trailing_store();
 1691   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1692 #ifdef ASSERT
 1693   if (release) {
 1694     Node* leading = mbvol->leading_membar();
 1695     assert(leading->Opcode() == Op_MemBarRelease, "");
 1696     assert(leading->as_MemBar()->leading_store(), "");
 1697     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1698   }
 1699 #endif
 1700 
 1701   return release;
 1702 }
 1703 
 1704 // predicates controlling emit of str<x>/stlr<x>
 1705 
 1706 bool needs_releasing_store(const Node *n)
 1707 {
 1708   // assert n->is_Store();
 1709   StoreNode *st = n->as_Store();
 1710   return st->trailing_membar() != NULL;
 1711 }
 1712 
 1713 // predicate controlling translation of CAS
 1714 //
 1715 // returns true if CAS needs to use an acquiring load otherwise false
 1716 
 1717 bool needs_acquiring_load_exclusive(const Node *n)
 1718 {
 1719   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1720   LoadStoreNode* ldst = n->as_LoadStore();
 1721   if (is_CAS(n->Opcode(), false)) {
 1722     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1723   } else {
 1724     return ldst->trailing_membar() != NULL;
 1725   }
 1726 
 1727   // so we can just return true here
 1728   return true;
 1729 }
 1730 
 1731 #define __ _masm.
 1732 
 1733 // advance declarations for helper functions to convert register
 1734 // indices to register objects
 1735 
 1736 // the ad file has to provide implementations of certain methods
 1737 // expected by the generic code
 1738 //
 1739 // REQUIRED FUNCTIONALITY
 1740 
 1741 //=============================================================================
 1742 
 1743 // !!!!! Special hack to get all types of calls to specify the byte offset
 1744 //       from the start of the call to the point where the return address
 1745 //       will point.
 1746 
 1747 int MachCallStaticJavaNode::ret_addr_offset()
 1748 {
 1749   // call should be a simple bl
 1750   int off = 4;
 1751   return off;
 1752 }
 1753 
 1754 int MachCallDynamicJavaNode::ret_addr_offset()
 1755 {
 1756   return 16; // movz, movk, movk, bl
 1757 }
 1758 
 1759 int MachCallRuntimeNode::ret_addr_offset() {
 1760   // for generated stubs the call will be
 1761   //   bl(addr)
 1762   // or with far branches
 1763   //   bl(trampoline_stub)
 1764   // for real runtime callouts it will be six instructions
 1765   // see aarch64_enc_java_to_runtime
 1766   //   adr(rscratch2, retaddr)
 1767   //   lea(rscratch1, RuntimeAddress(addr)
 1768   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1769   //   blr(rscratch1)
 1770   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1771   if (cb) {
 1772     return 1 * NativeInstruction::instruction_size;
 1773   } else {
 1774     return 6 * NativeInstruction::instruction_size;
 1775   }
 1776 }
 1777 
 1778 int MachCallNativeNode::ret_addr_offset() {
 1779   // This is implemented using aarch64_enc_java_to_runtime as above.
 1780   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1781   if (cb) {
 1782     return 1 * NativeInstruction::instruction_size;
 1783   } else {
 1784     return 6 * NativeInstruction::instruction_size;
 1785   }
 1786 }
 1787 
 1788 //=============================================================================
 1789 
 1790 #ifndef PRODUCT
 1791 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1792   st->print("BREAKPOINT");
 1793 }
 1794 #endif
 1795 
 1796 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1797   C2_MacroAssembler _masm(&cbuf);
 1798   __ brk(0);
 1799 }
 1800 
 1801 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1802   return MachNode::size(ra_);
 1803 }
 1804 
 1805 //=============================================================================
 1806 
 1807 #ifndef PRODUCT
 1808   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1809     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1810   }
 1811 #endif
 1812 
 1813   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1814     C2_MacroAssembler _masm(&cbuf);
 1815     for (int i = 0; i < _count; i++) {
 1816       __ nop();
 1817     }
 1818   }
 1819 
 1820   uint MachNopNode::size(PhaseRegAlloc*) const {
 1821     return _count * NativeInstruction::instruction_size;
 1822   }
 1823 
 1824 //=============================================================================
 1825 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1826 
 1827 int ConstantTable::calculate_table_base_offset() const {
 1828   return 0;  // absolute addressing, no offset
 1829 }
 1830 
 1831 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1832 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1833   ShouldNotReachHere();
 1834 }
 1835 
 1836 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1837   // Empty encoding
 1838 }
 1839 
 1840 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1841   return 0;
 1842 }
 1843 
 1844 #ifndef PRODUCT
 1845 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1846   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1847 }
 1848 #endif
 1849 
 1850 #ifndef PRODUCT
 1851 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1852   Compile* C = ra_->C;
 1853 
 1854   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1855 
 1856   if (C->output()->need_stack_bang(framesize))
 1857     st->print("# stack bang size=%d\n\t", framesize);
 1858 
 1859   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1860     st->print("sub  sp, sp, #%d\n\t", framesize);
 1861     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1862     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1863   } else {
 1864     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1865     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1866     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1867     st->print("sub  sp, sp, rscratch1");
 1868   }
 1869   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1870     st->print("\n\t");
 1871     st->print("ldr  rscratch1, [guard]\n\t");
 1872     st->print("dmb ishld\n\t");
 1873     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1874     st->print("cmp  rscratch1, rscratch2\n\t");
 1875     st->print("b.eq skip");
 1876     st->print("\n\t");
 1877     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1878     st->print("b skip\n\t");
 1879     st->print("guard: int\n\t");
 1880     st->print("\n\t");
 1881     st->print("skip:\n\t");
 1882   }
 1883 }
 1884 #endif
 1885 
 1886 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1887   Compile* C = ra_->C;
 1888   C2_MacroAssembler _masm(&cbuf);
 1889 
 1890   // n.b. frame size includes space for return pc and rfp
 1891   const int framesize = C->output()->frame_size_in_bytes();
 1892 
 1893   // insert a nop at the start of the prolog so we can patch in a
 1894   // branch if we need to invalidate the method later
 1895   __ nop();
 1896 
 1897   if (C->clinit_barrier_on_entry()) {
 1898     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1899 
 1900     Label L_skip_barrier;
 1901 
 1902     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1903     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1904     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1905     __ bind(L_skip_barrier);
 1906   }
 1907 
 1908   if (C->max_vector_size() > 0) {
 1909     __ reinitialize_ptrue();
 1910   }
 1911 
 1912   int bangsize = C->output()->bang_size_in_bytes();
 1913   if (C->output()->need_stack_bang(bangsize))
 1914     __ generate_stack_overflow_check(bangsize);
 1915 
 1916   __ build_frame(framesize);
 1917 
 1918   if (C->stub_function() == NULL) {
 1919     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1920     bs->nmethod_entry_barrier(&_masm);
 1921   }
 1922 
 1923   if (VerifyStackAtCalls) {
 1924     Unimplemented();
 1925   }
 1926 
 1927   C->output()->set_frame_complete(cbuf.insts_size());
 1928 
 1929   if (C->has_mach_constant_base_node()) {
 1930     // NOTE: We set the table base offset here because users might be
 1931     // emitted before MachConstantBaseNode.
 1932     ConstantTable& constant_table = C->output()->constant_table();
 1933     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1934   }
 1935 }
 1936 
 1937 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1938 {
 1939   return MachNode::size(ra_); // too many variables; just compute it
 1940                               // the hard way
 1941 }
 1942 
 1943 int MachPrologNode::reloc() const
 1944 {
 1945   return 0;
 1946 }
 1947 
 1948 //=============================================================================
 1949 
 1950 #ifndef PRODUCT
 1951 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1952   Compile* C = ra_->C;
 1953   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1954 
 1955   st->print("# pop frame %d\n\t",framesize);
 1956 
 1957   if (framesize == 0) {
 1958     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1959   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1960     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, #%d\n\t", framesize);
 1962   } else {
 1963     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1964     st->print("add  sp, sp, rscratch1\n\t");
 1965     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1966   }
 1967 
 1968   if (do_polling() && C->is_method_compilation()) {
 1969     st->print("# test polling word\n\t");
 1970     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1971     st->print("cmp  sp, rscratch1\n\t");
 1972     st->print("bhi #slow_path");
 1973   }
 1974 }
 1975 #endif
 1976 
 1977 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1978   Compile* C = ra_->C;
 1979   C2_MacroAssembler _masm(&cbuf);
 1980   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1981 
 1982   __ remove_frame(framesize);
 1983 
 1984   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1985     __ reserved_stack_check();
 1986   }
 1987 
 1988   if (do_polling() && C->is_method_compilation()) {
 1989     Label dummy_label;
 1990     Label* code_stub = &dummy_label;
 1991     if (!C->output()->in_scratch_emit_size()) {
 1992       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1993     }
 1994     __ relocate(relocInfo::poll_return_type);
 1995     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1996   }
 1997 }
 1998 
 1999 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 2000   // Variable size. Determine dynamically.
 2001   return MachNode::size(ra_);
 2002 }
 2003 
 2004 int MachEpilogNode::reloc() const {
 2005   // Return number of relocatable values contained in this instruction.
 2006   return 1; // 1 for polling page.
 2007 }
 2008 
 2009 const Pipeline * MachEpilogNode::pipeline() const {
 2010   return MachNode::pipeline_class();
 2011 }
 2012 
 2013 //=============================================================================
 2014 
 2015 // Figure out which register class each belongs in: rc_int, rc_float or
 2016 // rc_stack.
 2017 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2018 
 2019 static enum RC rc_class(OptoReg::Name reg) {
 2020 
 2021   if (reg == OptoReg::Bad) {
 2022     return rc_bad;
 2023   }
 2024 
 2025   // we have 32 int registers * 2 halves
 2026   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2027 
 2028   if (reg < slots_of_int_registers) {
 2029     return rc_int;
 2030   }
 2031 
 2032   // we have 32 float register * 8 halves
 2033   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2034   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2035     return rc_float;
 2036   }
 2037 
 2038   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2039   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2040     return rc_predicate;
 2041   }
 2042 
 2043   // Between predicate regs & stack is the flags.
 2044   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2045 
 2046   return rc_stack;
 2047 }
 2048 
 2049 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2050   Compile* C = ra_->C;
 2051 
 2052   // Get registers to move.
 2053   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2054   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2055   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2056   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2057 
 2058   enum RC src_hi_rc = rc_class(src_hi);
 2059   enum RC src_lo_rc = rc_class(src_lo);
 2060   enum RC dst_hi_rc = rc_class(dst_hi);
 2061   enum RC dst_lo_rc = rc_class(dst_lo);
 2062 
 2063   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2064 
 2065   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 2066     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2067            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2068            "expected aligned-adjacent pairs");
 2069   }
 2070 
 2071   if (src_lo == dst_lo && src_hi == dst_hi) {
 2072     return 0;            // Self copy, no move.
 2073   }
 2074 
 2075   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2076               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2077   int src_offset = ra_->reg2offset(src_lo);
 2078   int dst_offset = ra_->reg2offset(dst_lo);
 2079 
 2080   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2081     uint ireg = ideal_reg();
 2082     if (ireg == Op_VecA && cbuf) {
 2083       C2_MacroAssembler _masm(cbuf);
 2084       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2085       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2086         // stack->stack
 2087         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2088                                                 sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2090         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2091                             sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2093         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2094                               sve_vector_reg_size_in_bytes);
 2095       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2096         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2097                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2098                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2099       } else {
 2100         ShouldNotReachHere();
 2101       }
 2102     } else if (cbuf) {
 2103       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2104       C2_MacroAssembler _masm(cbuf);
 2105       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2106       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2107         // stack->stack
 2108         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2109         if (ireg == Op_VecD) {
 2110           __ unspill(rscratch1, true, src_offset);
 2111           __ spill(rscratch1, true, dst_offset);
 2112         } else {
 2113           __ spill_copy128(src_offset, dst_offset);
 2114         }
 2115       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2116         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2117                ireg == Op_VecD ? __ T8B : __ T16B,
 2118                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2119       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2120         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2121                  ireg == Op_VecD ? __ D : __ Q,
 2122                  ra_->reg2offset(dst_lo));
 2123       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2124         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2125                    ireg == Op_VecD ? __ D : __ Q,
 2126                    ra_->reg2offset(src_lo));
 2127       } else {
 2128         ShouldNotReachHere();
 2129       }
 2130     }
 2131   } else if (cbuf) {
 2132     C2_MacroAssembler _masm(cbuf);
 2133     switch (src_lo_rc) {
 2134     case rc_int:
 2135       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2136         if (is64) {
 2137             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2138                    as_Register(Matcher::_regEncode[src_lo]));
 2139         } else {
 2140             C2_MacroAssembler _masm(cbuf);
 2141             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2142                     as_Register(Matcher::_regEncode[src_lo]));
 2143         }
 2144       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2145         if (is64) {
 2146             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         } else {
 2149             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2150                      as_Register(Matcher::_regEncode[src_lo]));
 2151         }
 2152       } else {                    // gpr --> stack spill
 2153         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2154         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2155       }
 2156       break;
 2157     case rc_float:
 2158       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2159         if (is64) {
 2160             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         } else {
 2163             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2164                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2165         }
 2166       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2167         if (is64) {
 2168             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         } else {
 2171             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2172                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2173         }
 2174       } else {                    // fpr --> stack spill
 2175         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2176         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2177                  is64 ? __ D : __ S, dst_offset);
 2178       }
 2179       break;
 2180     case rc_stack:
 2181       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2182         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2183       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2184         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2185                    is64 ? __ D : __ S, src_offset);
 2186       } else if (dst_lo_rc == rc_predicate) {
 2187         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2188                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2189       } else {                    // stack --> stack copy
 2190         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2191         if (ideal_reg() == Op_RegVectMask) {
 2192           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2193                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2194         } else {
 2195           __ unspill(rscratch1, is64, src_offset);
 2196           __ spill(rscratch1, is64, dst_offset);
 2197         }
 2198       }
 2199       break;
 2200     case rc_predicate:
 2201       if (dst_lo_rc == rc_predicate) {
 2202         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2203       } else if (dst_lo_rc == rc_stack) {
 2204         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2205                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2206       } else {
 2207         assert(false, "bad src and dst rc_class combination.");
 2208         ShouldNotReachHere();
 2209       }
 2210       break;
 2211     default:
 2212       assert(false, "bad rc_class for spill");
 2213       ShouldNotReachHere();
 2214     }
 2215   }
 2216 
 2217   if (st) {
 2218     st->print("spill ");
 2219     if (src_lo_rc == rc_stack) {
 2220       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2221     } else {
 2222       st->print("%s -> ", Matcher::regName[src_lo]);
 2223     }
 2224     if (dst_lo_rc == rc_stack) {
 2225       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2226     } else {
 2227       st->print("%s", Matcher::regName[dst_lo]);
 2228     }
 2229     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2230       int vsize = 0;
 2231       switch (ideal_reg()) {
 2232       case Op_VecD:
 2233         vsize = 64;
 2234         break;
 2235       case Op_VecX:
 2236         vsize = 128;
 2237         break;
 2238       case Op_VecA:
 2239         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2240         break;
 2241       default:
 2242         assert(false, "bad register type for spill");
 2243         ShouldNotReachHere();
 2244       }
 2245       st->print("\t# vector spill size = %d", vsize);
 2246     } else if (ideal_reg() == Op_RegVectMask) {
 2247       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2248       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2249       st->print("\t# predicate spill size = %d", vsize);
 2250     } else {
 2251       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2252     }
 2253   }
 2254 
 2255   return 0;
 2256 
 2257 }
 2258 
 2259 #ifndef PRODUCT
 2260 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2261   if (!ra_)
 2262     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2263   else
 2264     implementation(NULL, ra_, false, st);
 2265 }
 2266 #endif
 2267 
 2268 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2269   implementation(&cbuf, ra_, false, NULL);
 2270 }
 2271 
 2272 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2273   return MachNode::size(ra_);
 2274 }
 2275 
 2276 //=============================================================================
 2277 
 2278 #ifndef PRODUCT
 2279 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2280   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2281   int reg = ra_->get_reg_first(this);
 2282   st->print("add %s, rsp, #%d]\t# box lock",
 2283             Matcher::regName[reg], offset);
 2284 }
 2285 #endif
 2286 
 2287 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2288   C2_MacroAssembler _masm(&cbuf);
 2289 
 2290   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2291   int reg    = ra_->get_encode(this);
 2292 
 2293   // This add will handle any 24-bit signed offset. 24 bits allows an
 2294   // 8 megabyte stack frame.
 2295   __ add(as_Register(reg), sp, offset);
 2296 }
 2297 
 2298 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2299   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2300   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2301 
 2302   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2303     return NativeInstruction::instruction_size;
 2304   } else {
 2305     return 2 * NativeInstruction::instruction_size;
 2306   }
 2307 }
 2308 
 2309 //=============================================================================
 2310 
 2311 #ifndef PRODUCT
 2312 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2313 {
 2314   st->print_cr("# MachUEPNode");
 2315   if (UseCompressedClassPointers) {
 2316     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2317     if (CompressedKlassPointers::shift() != 0) {
 2318       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2319     }
 2320   } else {
 2321    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2322   }
 2323   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2324   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2325 }
 2326 #endif
 2327 
 2328 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2329 {
 2330   // This is the unverified entry point.
 2331   C2_MacroAssembler _masm(&cbuf);
 2332 
 2333   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2334   Label skip;
 2335   // TODO
 2336   // can we avoid this skip and still use a reloc?
 2337   __ br(Assembler::EQ, skip);
 2338   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2339   __ bind(skip);
 2340 }
 2341 
 2342 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2343 {
 2344   return MachNode::size(ra_);
 2345 }
 2346 
 2347 // REQUIRED EMIT CODE
 2348 
 2349 //=============================================================================
 2350 
 2351 // Emit exception handler code.
 2352 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2353 {
 2354   // mov rscratch1 #exception_blob_entry_point
 2355   // br rscratch1
 2356   // Note that the code buffer's insts_mark is always relative to insts.
 2357   // That's why we must use the macroassembler to generate a handler.
 2358   C2_MacroAssembler _masm(&cbuf);
 2359   address base = __ start_a_stub(size_exception_handler());
 2360   if (base == NULL) {
 2361     ciEnv::current()->record_failure("CodeCache is full");
 2362     return 0;  // CodeBuffer::expand failed
 2363   }
 2364   int offset = __ offset();
 2365   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2366   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2367   __ end_a_stub();
 2368   return offset;
 2369 }
 2370 
 2371 // Emit deopt handler code.
 2372 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2373 {
 2374   // Note that the code buffer's insts_mark is always relative to insts.
 2375   // That's why we must use the macroassembler to generate a handler.
 2376   C2_MacroAssembler _masm(&cbuf);
 2377   address base = __ start_a_stub(size_deopt_handler());
 2378   if (base == NULL) {
 2379     ciEnv::current()->record_failure("CodeCache is full");
 2380     return 0;  // CodeBuffer::expand failed
 2381   }
 2382   int offset = __ offset();
 2383 
 2384   __ adr(lr, __ pc());
 2385   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2386 
 2387   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2388   __ end_a_stub();
 2389   return offset;
 2390 }
 2391 
 2392 // REQUIRED MATCHER CODE
 2393 
 2394 //=============================================================================
 2395 
 2396 const bool Matcher::match_rule_supported(int opcode) {
 2397   if (!has_match_rule(opcode))
 2398     return false;
 2399 
 2400   bool ret_value = true;
 2401   switch (opcode) {
 2402     case Op_OnSpinWait:
 2403       return VM_Version::supports_on_spin_wait();
 2404     case Op_CacheWB:
 2405     case Op_CacheWBPreSync:
 2406     case Op_CacheWBPostSync:
 2407       if (!VM_Version::supports_data_cache_line_flush()) {
 2408         ret_value = false;
 2409       }
 2410       break;
 2411     case Op_LoadVectorMasked:
 2412     case Op_StoreVectorMasked:
 2413     case Op_LoadVectorGatherMasked:
 2414     case Op_StoreVectorScatterMasked:
 2415     case Op_MaskAll:
 2416     case Op_AndVMask:
 2417     case Op_OrVMask:
 2418     case Op_XorVMask:
 2419       if (UseSVE == 0) {
 2420         ret_value = false;
 2421       }
 2422       break;
 2423   }
 2424 
 2425   return ret_value; // Per default match rules are supported.
 2426 }
 2427 
 2428 // Identify extra cases that we might want to provide match rules for vector nodes and
 2429 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2430 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2431   if (!match_rule_supported(opcode)) {
 2432     return false;
 2433   }
 2434   int bit_size = vlen * type2aelembytes(bt) * 8;
 2435   if (UseSVE == 0 && bit_size > 128) {
 2436     return false;
 2437   }
 2438   if (UseSVE > 0) {
 2439     return op_sve_supported(opcode, vlen, bt);
 2440   } else { // NEON
 2441     // Special cases
 2442     switch (opcode) {
 2443     case Op_VectorMaskCmp:
 2444       if (vlen < 2 || bit_size < 64) {
 2445         return false;
 2446       }
 2447       break;
 2448     case Op_MulAddVS2VI:
 2449       if (bit_size < 128) {
 2450         return false;
 2451       }
 2452       break;
 2453     case Op_MulVL:
 2454       return false;
 2455     case Op_VectorLoadShuffle:
 2456     case Op_VectorRearrange:
 2457       if (vlen < 4) {
 2458         return false;
 2459       }
 2460       break;
 2461     case Op_LoadVectorGather:
 2462     case Op_StoreVectorScatter:
 2463     case Op_CompressV:
 2464     case Op_CompressM:
 2465       return false;
 2466     default:
 2467       break;
 2468     }
 2469   }
 2470   return vector_size_supported(bt, vlen);
 2471 }
 2472 
 2473 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
 2474   // Only SVE supports masked operations.
 2475   if (UseSVE == 0) {
 2476     return false;
 2477   }
 2478   return match_rule_supported(opcode) &&
 2479          masked_op_sve_supported(opcode, vlen, bt);
 2480 }
 2481 
 2482 const RegMask* Matcher::predicate_reg_mask(void) {
 2483   return &_PR_REG_mask;
 2484 }
 2485 
 2486 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2487   return new TypeVectMask(elemTy, length);
 2488 }
 2489 
 2490 // Vector calling convention not yet implemented.
 2491 const bool Matcher::supports_vector_calling_convention(void) {
 2492   return false;
 2493 }
 2494 
 2495 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2496   Unimplemented();
 2497   return OptoRegPair(0, 0);
 2498 }
 2499 
 2500 // Is this branch offset short enough that a short branch can be used?
 2501 //
 2502 // NOTE: If the platform does not provide any short branch variants, then
 2503 //       this method should return false for offset 0.
 2504 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2505   // The passed offset is relative to address of the branch.
 2506 
 2507   return (-32768 <= offset && offset < 32768);
 2508 }
 2509 
 2510 // Vector width in bytes.
 2511 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2512   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2513   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2514   // Minimum 2 values in vector
 2515   if (size < 2*type2aelembytes(bt)) size = 0;
 2516   // But never < 4
 2517   if (size < 4) size = 0;
 2518   return size;
 2519 }
 2520 
 2521 // Limits on vector size (number of elements) loaded into vector.
 2522 const int Matcher::max_vector_size(const BasicType bt) {
 2523   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2524 }
 2525 
 2526 const int Matcher::min_vector_size(const BasicType bt) {
 2527   int max_size = max_vector_size(bt);
 2528   // Limit the min vector size to 8 bytes.
 2529   int size = 8 / type2aelembytes(bt);
 2530   if (bt == T_BYTE) {
 2531     // To support vector api shuffle/rearrange.
 2532     size = 4;
 2533   } else if (bt == T_BOOLEAN) {
 2534     // To support vector api load/store mask.
 2535     size = 2;
 2536   }
 2537   if (size < 2) size = 2;
 2538   return MIN2(size, max_size);
 2539 }
 2540 
 2541 // Actual max scalable vector register length.
 2542 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2543   return Matcher::max_vector_size(bt);
 2544 }
 2545 
 2546 // Vector ideal reg.
 2547 const uint Matcher::vector_ideal_reg(int len) {
 2548   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2549     return Op_VecA;
 2550   }
 2551   switch(len) {
 2552     // For 16-bit/32-bit mask vector, reuse VecD.
 2553     case  2:
 2554     case  4:
 2555     case  8: return Op_VecD;
 2556     case 16: return Op_VecX;
 2557   }
 2558   ShouldNotReachHere();
 2559   return 0;
 2560 }
 2561 
 2562 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2563   ShouldNotReachHere(); // generic vector operands not supported
 2564   return NULL;
 2565 }
 2566 
 2567 bool Matcher::is_reg2reg_move(MachNode* m) {
 2568   ShouldNotReachHere();  // generic vector operands not supported
 2569   return false;
 2570 }
 2571 
 2572 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2573   ShouldNotReachHere();  // generic vector operands not supported
 2574   return false;
 2575 }
 2576 
 2577 // Return whether or not this register is ever used as an argument.
 2578 // This function is used on startup to build the trampoline stubs in
 2579 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2580 // call in the trampoline, and arguments in those registers not be
 2581 // available to the callee.
 2582 bool Matcher::can_be_java_arg(int reg)
 2583 {
 2584   return
 2585     reg ==  R0_num || reg == R0_H_num ||
 2586     reg ==  R1_num || reg == R1_H_num ||
 2587     reg ==  R2_num || reg == R2_H_num ||
 2588     reg ==  R3_num || reg == R3_H_num ||
 2589     reg ==  R4_num || reg == R4_H_num ||
 2590     reg ==  R5_num || reg == R5_H_num ||
 2591     reg ==  R6_num || reg == R6_H_num ||
 2592     reg ==  R7_num || reg == R7_H_num ||
 2593     reg ==  V0_num || reg == V0_H_num ||
 2594     reg ==  V1_num || reg == V1_H_num ||
 2595     reg ==  V2_num || reg == V2_H_num ||
 2596     reg ==  V3_num || reg == V3_H_num ||
 2597     reg ==  V4_num || reg == V4_H_num ||
 2598     reg ==  V5_num || reg == V5_H_num ||
 2599     reg ==  V6_num || reg == V6_H_num ||
 2600     reg ==  V7_num || reg == V7_H_num;
 2601 }
 2602 
 2603 bool Matcher::is_spillable_arg(int reg)
 2604 {
 2605   return can_be_java_arg(reg);
 2606 }
 2607 
 2608 uint Matcher::int_pressure_limit()
 2609 {
 2610   // JDK-8183543: When taking the number of available registers as int
 2611   // register pressure threshold, the jtreg test:
 2612   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2613   // failed due to C2 compilation failure with
 2614   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2615   //
 2616   // A derived pointer is live at CallNode and then is flagged by RA
 2617   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2618   // derived pointers and lastly fail to spill after reaching maximum
 2619   // number of iterations. Lowering the default pressure threshold to
 2620   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2621   // a high register pressure area of the code so that split_DEF can
 2622   // generate DefinitionSpillCopy for the derived pointer.
 2623   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2624   if (!PreserveFramePointer) {
 2625     // When PreserveFramePointer is off, frame pointer is allocatable,
 2626     // but different from other SOC registers, it is excluded from
 2627     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2628     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2629     // See check_pressure_at_fatproj().
 2630     default_int_pressure_threshold--;
 2631   }
 2632   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2633 }
 2634 
 2635 uint Matcher::float_pressure_limit()
 2636 {
 2637   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2638   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2639 }
 2640 
 2641 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2642   return false;
 2643 }
 2644 
 2645 RegMask Matcher::divI_proj_mask() {
 2646   ShouldNotReachHere();
 2647   return RegMask();
 2648 }
 2649 
 2650 // Register for MODI projection of divmodI.
 2651 RegMask Matcher::modI_proj_mask() {
 2652   ShouldNotReachHere();
 2653   return RegMask();
 2654 }
 2655 
 2656 // Register for DIVL projection of divmodL.
 2657 RegMask Matcher::divL_proj_mask() {
 2658   ShouldNotReachHere();
 2659   return RegMask();
 2660 }
 2661 
 2662 // Register for MODL projection of divmodL.
 2663 RegMask Matcher::modL_proj_mask() {
 2664   ShouldNotReachHere();
 2665   return RegMask();
 2666 }
 2667 
 2668 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2669   return FP_REG_mask();
 2670 }
 2671 
 2672 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2673   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2674     Node* u = addp->fast_out(i);
 2675     if (u->is_LoadStore()) {
 2676       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2677       // instructions) only take register indirect as an operand, so
 2678       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2679       // must fail.
 2680       return false;
 2681     }
 2682     if (u->is_Mem()) {
 2683       int opsize = u->as_Mem()->memory_size();
 2684       assert(opsize > 0, "unexpected memory operand size");
 2685       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2686         return false;
 2687       }
 2688     }
 2689   }
 2690   return true;
 2691 }
 2692 
 2693 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
 2694   if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
 2695     return false;
 2696   }
 2697   Node* imm_node = replicate_node->in(1);
 2698   if (!imm_node->is_Con()) {
 2699     return false;
 2700   }
 2701 
 2702   const Type* t = imm_node->bottom_type();
 2703   if (!(t->isa_int() || t->isa_long())) {
 2704     return false;
 2705   }
 2706 
 2707   switch (binary_node->Opcode()) {
 2708   case Op_AndV:
 2709   case Op_OrV:
 2710   case Op_XorV: {
 2711     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
 2712     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2713     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2714   }
 2715   case Op_AddVB:
 2716     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2717   case Op_AddVS:
 2718   case Op_AddVI:
 2719     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2720   case Op_AddVL:
 2721     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2722   default:
 2723     return false;
 2724   }
 2725 }
 2726 
 2727 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
 2728   if (n != NULL && m != NULL) {
 2729     return can_combine_with_imm(n, m);
 2730   }
 2731   return false;
 2732 }
 2733 
 2734 // Should the matcher clone input 'm' of node 'n'?
 2735 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2736   // ShiftV src (ShiftCntV con)
 2737   // StoreVector (VectorStoreMask src)
 2738   // Binary src (Replicate con)
 2739   if (is_vshift_con_pattern(n, m) ||
 2740       (UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector) ||
 2741       is_vector_arith_imm_pattern(n, m)) {
 2742     mstack.push(m, Visit);
 2743     return true;
 2744   }
 2745 
 2746   return false;
 2747 }
 2748 
 2749 // Should the Matcher clone shifts on addressing modes, expecting them
 2750 // to be subsumed into complex addressing expressions or compute them
 2751 // into registers?
 2752 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2753   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2754     return true;
 2755   }
 2756 
 2757   Node *off = m->in(AddPNode::Offset);
 2758   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2759       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2760       // Are there other uses besides address expressions?
 2761       !is_visited(off)) {
 2762     address_visited.set(off->_idx); // Flag as address_visited
 2763     mstack.push(off->in(2), Visit);
 2764     Node *conv = off->in(1);
 2765     if (conv->Opcode() == Op_ConvI2L &&
 2766         // Are there other uses besides address expressions?
 2767         !is_visited(conv)) {
 2768       address_visited.set(conv->_idx); // Flag as address_visited
 2769       mstack.push(conv->in(1), Pre_Visit);
 2770     } else {
 2771       mstack.push(conv, Pre_Visit);
 2772     }
 2773     address_visited.test_set(m->_idx); // Flag as address_visited
 2774     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2775     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2776     return true;
 2777   } else if (off->Opcode() == Op_ConvI2L &&
 2778              // Are there other uses besides address expressions?
 2779              !is_visited(off)) {
 2780     address_visited.test_set(m->_idx); // Flag as address_visited
 2781     address_visited.set(off->_idx); // Flag as address_visited
 2782     mstack.push(off->in(1), Pre_Visit);
 2783     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2784     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2785     return true;
 2786   }
 2787   return false;
 2788 }
 2789 
 2790 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2791   C2_MacroAssembler _masm(&cbuf);                                       \
 2792   {                                                                     \
 2793     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2794     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2795     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2796     __ INSN(REG, as_Register(BASE));                                    \
 2797   }
 2798 
 2799 
 2800 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2801   {
 2802     Address::extend scale;
 2803 
 2804     // Hooboy, this is fugly.  We need a way to communicate to the
 2805     // encoder that the index needs to be sign extended, so we have to
 2806     // enumerate all the cases.
 2807     switch (opcode) {
 2808     case INDINDEXSCALEDI2L:
 2809     case INDINDEXSCALEDI2LN:
 2810     case INDINDEXI2L:
 2811     case INDINDEXI2LN:
 2812       scale = Address::sxtw(size);
 2813       break;
 2814     default:
 2815       scale = Address::lsl(size);
 2816     }
 2817 
 2818     if (index == -1) {
 2819       return Address(base, disp);
 2820     } else {
 2821       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2822       return Address(base, as_Register(index), scale);
 2823     }
 2824   }
 2825 
 2826 
 2827 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2828 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2829 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2830 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2831                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2832 
 2833   // Used for all non-volatile memory accesses.  The use of
 2834   // $mem->opcode() to discover whether this pattern uses sign-extended
 2835   // offsets is something of a kludge.
 2836   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2837                         Register reg, int opcode,
 2838                         Register base, int index, int scale, int disp,
 2839                         int size_in_memory)
 2840   {
 2841     Address addr = mem2address(opcode, base, index, scale, disp);
 2842     if (addr.getMode() == Address::base_plus_offset) {
 2843       /* If we get an out-of-range offset it is a bug in the compiler,
 2844          so we assert here. */
 2845       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2846              "c2 compiler bug");
 2847       /* Fix up any out-of-range offsets. */
 2848       assert_different_registers(rscratch1, base);
 2849       assert_different_registers(rscratch1, reg);
 2850       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2851     }
 2852     (masm.*insn)(reg, addr);
 2853   }
 2854 
 2855   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2856                         FloatRegister reg, int opcode,
 2857                         Register base, int index, int size, int disp,
 2858                         int size_in_memory)
 2859   {
 2860     Address::extend scale;
 2861 
 2862     switch (opcode) {
 2863     case INDINDEXSCALEDI2L:
 2864     case INDINDEXSCALEDI2LN:
 2865       scale = Address::sxtw(size);
 2866       break;
 2867     default:
 2868       scale = Address::lsl(size);
 2869     }
 2870 
 2871     if (index == -1) {
 2872       /* If we get an out-of-range offset it is a bug in the compiler,
 2873          so we assert here. */
 2874       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2875       /* Fix up any out-of-range offsets. */
 2876       assert_different_registers(rscratch1, base);
 2877       Address addr = Address(base, disp);
 2878       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2879       (masm.*insn)(reg, addr);
 2880     } else {
 2881       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2882       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2883     }
 2884   }
 2885 
 2886   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2887                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2888                         int opcode, Register base, int index, int size, int disp)
 2889   {
 2890     if (index == -1) {
 2891       (masm.*insn)(reg, T, Address(base, disp));
 2892     } else {
 2893       assert(disp == 0, "unsupported address mode");
 2894       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2895     }
 2896   }
 2897 
 2898 %}
 2899 
 2900 
 2901 
 2902 //----------ENCODING BLOCK-----------------------------------------------------
 2903 // This block specifies the encoding classes used by the compiler to
 2904 // output byte streams.  Encoding classes are parameterized macros
 2905 // used by Machine Instruction Nodes in order to generate the bit
 2906 // encoding of the instruction.  Operands specify their base encoding
 2907 // interface with the interface keyword.  There are currently
 2908 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2909 // COND_INTER.  REG_INTER causes an operand to generate a function
 2910 // which returns its register number when queried.  CONST_INTER causes
 2911 // an operand to generate a function which returns the value of the
 2912 // constant when queried.  MEMORY_INTER causes an operand to generate
 2913 // four functions which return the Base Register, the Index Register,
 2914 // the Scale Value, and the Offset Value of the operand when queried.
 2915 // COND_INTER causes an operand to generate six functions which return
 2916 // the encoding code (ie - encoding bits for the instruction)
 2917 // associated with each basic boolean condition for a conditional
 2918 // instruction.
 2919 //
 2920 // Instructions specify two basic values for encoding.  Again, a
 2921 // function is available to check if the constant displacement is an
 2922 // oop. They use the ins_encode keyword to specify their encoding
 2923 // classes (which must be a sequence of enc_class names, and their
 2924 // parameters, specified in the encoding block), and they use the
 2925 // opcode keyword to specify, in order, their primary, secondary, and
 2926 // tertiary opcode.  Only the opcode sections which a particular
 2927 // instruction needs for encoding need to be specified.
 2928 encode %{
 2929   // Build emit functions for each basic byte or larger field in the
 2930   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2931   // from C++ code in the enc_class source block.  Emit functions will
 2932   // live in the main source block for now.  In future, we can
 2933   // generalize this by adding a syntax that specifies the sizes of
 2934   // fields in an order, so that the adlc can build the emit functions
 2935   // automagically
 2936 
 2937   // catch all for unimplemented encodings
 2938   enc_class enc_unimplemented %{
 2939     C2_MacroAssembler _masm(&cbuf);
 2940     __ unimplemented("C2 catch all");
 2941   %}
 2942 
 2943   // BEGIN Non-volatile memory access
 2944 
 2945   // This encoding class is generated automatically from ad_encode.m4.
 2946   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2947   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2948     Register dst_reg = as_Register($dst$$reg);
 2949     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2950                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2951   %}
 2952 
 2953   // This encoding class is generated automatically from ad_encode.m4.
 2954   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2955   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2956     Register dst_reg = as_Register($dst$$reg);
 2957     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2958                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2959   %}
 2960 
 2961   // This encoding class is generated automatically from ad_encode.m4.
 2962   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2963   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2964     Register dst_reg = as_Register($dst$$reg);
 2965     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2967   %}
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2972     Register dst_reg = as_Register($dst$$reg);
 2973     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2980     Register dst_reg = as_Register($dst$$reg);
 2981     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2988     Register dst_reg = as_Register($dst$$reg);
 2989     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2991   %}
 2992 
 2993   // This encoding class is generated automatically from ad_encode.m4.
 2994   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2995   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2996     Register dst_reg = as_Register($dst$$reg);
 2997     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2998                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2999   %}
 3000 
 3001   // This encoding class is generated automatically from ad_encode.m4.
 3002   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3003   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 3004     Register dst_reg = as_Register($dst$$reg);
 3005     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 3006                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3007   %}
 3008 
 3009   // This encoding class is generated automatically from ad_encode.m4.
 3010   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3011   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 3012     Register dst_reg = as_Register($dst$$reg);
 3013     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3014                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3015   %}
 3016 
 3017   // This encoding class is generated automatically from ad_encode.m4.
 3018   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3019   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 3020     Register dst_reg = as_Register($dst$$reg);
 3021     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 3022                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3023   %}
 3024 
 3025   // This encoding class is generated automatically from ad_encode.m4.
 3026   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3027   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 3028     Register dst_reg = as_Register($dst$$reg);
 3029     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 3030                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3031   %}
 3032 
 3033   // This encoding class is generated automatically from ad_encode.m4.
 3034   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3035   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 3036     Register dst_reg = as_Register($dst$$reg);
 3037     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 3038                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3039   %}
 3040 
 3041   // This encoding class is generated automatically from ad_encode.m4.
 3042   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3043   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 3044     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3045     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 3046                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3047   %}
 3048 
 3049   // This encoding class is generated automatically from ad_encode.m4.
 3050   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3051   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3052     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3053     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3054                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3055   %}
 3056 
 3057   // This encoding class is generated automatically from ad_encode.m4.
 3058   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3059   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3060     Register src_reg = as_Register($src$$reg);
 3061     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3062                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3063   %}
 3064 
 3065   // This encoding class is generated automatically from ad_encode.m4.
 3066   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3067   enc_class aarch64_enc_strb0(memory1 mem) %{
 3068     C2_MacroAssembler _masm(&cbuf);
 3069     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3070                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3071   %}
 3072 
 3073   // This encoding class is generated automatically from ad_encode.m4.
 3074   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3075   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3076     Register src_reg = as_Register($src$$reg);
 3077     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3078                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3079   %}
 3080 
 3081   // This encoding class is generated automatically from ad_encode.m4.
 3082   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3083   enc_class aarch64_enc_strh0(memory2 mem) %{
 3084     C2_MacroAssembler _masm(&cbuf);
 3085     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3086                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3087   %}
 3088 
 3089   // This encoding class is generated automatically from ad_encode.m4.
 3090   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3091   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3092     Register src_reg = as_Register($src$$reg);
 3093     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3094                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3095   %}
 3096 
 3097   // This encoding class is generated automatically from ad_encode.m4.
 3098   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3099   enc_class aarch64_enc_strw0(memory4 mem) %{
 3100     C2_MacroAssembler _masm(&cbuf);
 3101     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3102                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3103   %}
 3104 
 3105   // This encoding class is generated automatically from ad_encode.m4.
 3106   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3107   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3108     Register src_reg = as_Register($src$$reg);
 3109     // we sometimes get asked to store the stack pointer into the
 3110     // current thread -- we cannot do that directly on AArch64
 3111     if (src_reg == r31_sp) {
 3112       C2_MacroAssembler _masm(&cbuf);
 3113       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3114       __ mov(rscratch2, sp);
 3115       src_reg = rscratch2;
 3116     }
 3117     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3118                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3119   %}
 3120 
 3121   // This encoding class is generated automatically from ad_encode.m4.
 3122   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3123   enc_class aarch64_enc_str0(memory8 mem) %{
 3124     C2_MacroAssembler _masm(&cbuf);
 3125     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3126                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3127   %}
 3128 
 3129   // This encoding class is generated automatically from ad_encode.m4.
 3130   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3131   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3132     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3133     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3134                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3135   %}
 3136 
 3137   // This encoding class is generated automatically from ad_encode.m4.
 3138   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3139   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3140     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3141     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3142                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3143   %}
 3144 
 3145   // This encoding class is generated automatically from ad_encode.m4.
 3146   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3147   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3148       C2_MacroAssembler _masm(&cbuf);
 3149       __ membar(Assembler::StoreStore);
 3150       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3151                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3152   %}
 3153 
 3154   // END Non-volatile memory access
 3155 
 3156   // Vector loads and stores
 3157   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3158     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3159     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3160        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3161   %}
 3162 
 3163   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3164     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3165     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3166        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3170     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3171     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3172        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3173   %}
 3174 
 3175   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3176     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3177     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3178        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3179   %}
 3180 
 3181   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3182     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3183     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3184        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3188     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3189     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3190        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3191   %}
 3192 
 3193   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3194     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3195     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3196        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3197   %}
 3198 
 3199   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3200     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3201     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3202        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3203   %}
 3204 
 3205   // volatile loads and stores
 3206 
 3207   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3208     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3209                  rscratch1, stlrb);
 3210   %}
 3211 
 3212   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3213     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3214                  rscratch1, stlrh);
 3215   %}
 3216 
 3217   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3218     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3219                  rscratch1, stlrw);
 3220   %}
 3221 
 3222 
 3223   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3224     Register dst_reg = as_Register($dst$$reg);
 3225     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3226              rscratch1, ldarb);
 3227     __ sxtbw(dst_reg, dst_reg);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3231     Register dst_reg = as_Register($dst$$reg);
 3232     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3233              rscratch1, ldarb);
 3234     __ sxtb(dst_reg, dst_reg);
 3235   %}
 3236 
 3237   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3238     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3239              rscratch1, ldarb);
 3240   %}
 3241 
 3242   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3243     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3244              rscratch1, ldarb);
 3245   %}
 3246 
 3247   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3248     Register dst_reg = as_Register($dst$$reg);
 3249     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3250              rscratch1, ldarh);
 3251     __ sxthw(dst_reg, dst_reg);
 3252   %}
 3253 
 3254   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3255     Register dst_reg = as_Register($dst$$reg);
 3256     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3257              rscratch1, ldarh);
 3258     __ sxth(dst_reg, dst_reg);
 3259   %}
 3260 
 3261   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3262     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3263              rscratch1, ldarh);
 3264   %}
 3265 
 3266   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3267     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3268              rscratch1, ldarh);
 3269   %}
 3270 
 3271   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3272     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3273              rscratch1, ldarw);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3277     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3278              rscratch1, ldarw);
 3279   %}
 3280 
 3281   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3282     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3283              rscratch1, ldar);
 3284   %}
 3285 
 3286   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3287     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3288              rscratch1, ldarw);
 3289     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3290   %}
 3291 
 3292   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3293     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3294              rscratch1, ldar);
 3295     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3296   %}
 3297 
 3298   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3299     Register src_reg = as_Register($src$$reg);
 3300     // we sometimes get asked to store the stack pointer into the
 3301     // current thread -- we cannot do that directly on AArch64
 3302     if (src_reg == r31_sp) {
 3303       C2_MacroAssembler _masm(&cbuf);
 3304       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3305       __ mov(rscratch2, sp);
 3306       src_reg = rscratch2;
 3307     }
 3308     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3309                  rscratch1, stlr);
 3310   %}
 3311 
 3312   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3313     {
 3314       C2_MacroAssembler _masm(&cbuf);
 3315       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3316       __ fmovs(rscratch2, src_reg);
 3317     }
 3318     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3319                  rscratch1, stlrw);
 3320   %}
 3321 
 3322   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3323     {
 3324       C2_MacroAssembler _masm(&cbuf);
 3325       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3326       __ fmovd(rscratch2, src_reg);
 3327     }
 3328     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3329                  rscratch1, stlr);
 3330   %}
 3331 
 3332   // synchronized read/update encodings
 3333 
 3334   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3335     C2_MacroAssembler _masm(&cbuf);
 3336     Register dst_reg = as_Register($dst$$reg);
 3337     Register base = as_Register($mem$$base);
 3338     int index = $mem$$index;
 3339     int scale = $mem$$scale;
 3340     int disp = $mem$$disp;
 3341     if (index == -1) {
 3342        if (disp != 0) {
 3343         __ lea(rscratch1, Address(base, disp));
 3344         __ ldaxr(dst_reg, rscratch1);
 3345       } else {
 3346         // TODO
 3347         // should we ever get anything other than this case?
 3348         __ ldaxr(dst_reg, base);
 3349       }
 3350     } else {
 3351       Register index_reg = as_Register(index);
 3352       if (disp == 0) {
 3353         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3354         __ ldaxr(dst_reg, rscratch1);
 3355       } else {
 3356         __ lea(rscratch1, Address(base, disp));
 3357         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3358         __ ldaxr(dst_reg, rscratch1);
 3359       }
 3360     }
 3361   %}
 3362 
 3363   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3364     C2_MacroAssembler _masm(&cbuf);
 3365     Register src_reg = as_Register($src$$reg);
 3366     Register base = as_Register($mem$$base);
 3367     int index = $mem$$index;
 3368     int scale = $mem$$scale;
 3369     int disp = $mem$$disp;
 3370     if (index == -1) {
 3371        if (disp != 0) {
 3372         __ lea(rscratch2, Address(base, disp));
 3373         __ stlxr(rscratch1, src_reg, rscratch2);
 3374       } else {
 3375         // TODO
 3376         // should we ever get anything other than this case?
 3377         __ stlxr(rscratch1, src_reg, base);
 3378       }
 3379     } else {
 3380       Register index_reg = as_Register(index);
 3381       if (disp == 0) {
 3382         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3383         __ stlxr(rscratch1, src_reg, rscratch2);
 3384       } else {
 3385         __ lea(rscratch2, Address(base, disp));
 3386         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3387         __ stlxr(rscratch1, src_reg, rscratch2);
 3388       }
 3389     }
 3390     __ cmpw(rscratch1, zr);
 3391   %}
 3392 
 3393   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3394     C2_MacroAssembler _masm(&cbuf);
 3395     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3396     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3397                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3398                /*weak*/ false, noreg);
 3399   %}
 3400 
 3401   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3402     C2_MacroAssembler _masm(&cbuf);
 3403     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3404     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3405                Assembler::word, /*acquire*/ false, /*release*/ true,
 3406                /*weak*/ false, noreg);
 3407   %}
 3408 
 3409   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3410     C2_MacroAssembler _masm(&cbuf);
 3411     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3412     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3413                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3414                /*weak*/ false, noreg);
 3415   %}
 3416 
 3417   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3418     C2_MacroAssembler _masm(&cbuf);
 3419     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3420     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3421                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3422                /*weak*/ false, noreg);
 3423   %}
 3424 
 3425 
 3426   // The only difference between aarch64_enc_cmpxchg and
 3427   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3428   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3429   // lock.
 3430   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3431     C2_MacroAssembler _masm(&cbuf);
 3432     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3433     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3434                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3435                /*weak*/ false, noreg);
 3436   %}
 3437 
 3438   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3439     C2_MacroAssembler _masm(&cbuf);
 3440     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3441     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3442                Assembler::word, /*acquire*/ true, /*release*/ true,
 3443                /*weak*/ false, noreg);
 3444   %}
 3445 
 3446   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3447     C2_MacroAssembler _masm(&cbuf);
 3448     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3449     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3450                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3451                /*weak*/ false, noreg);
 3452   %}
 3453 
 3454   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3455     C2_MacroAssembler _masm(&cbuf);
 3456     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3457     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3458                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3459                /*weak*/ false, noreg);
 3460   %}
 3461 
 3462   // auxiliary used for CompareAndSwapX to set result register
 3463   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3464     C2_MacroAssembler _masm(&cbuf);
 3465     Register res_reg = as_Register($res$$reg);
 3466     __ cset(res_reg, Assembler::EQ);
 3467   %}
 3468 
 3469   // prefetch encodings
 3470 
 3471   enc_class aarch64_enc_prefetchw(memory mem) %{
 3472     C2_MacroAssembler _masm(&cbuf);
 3473     Register base = as_Register($mem$$base);
 3474     int index = $mem$$index;
 3475     int scale = $mem$$scale;
 3476     int disp = $mem$$disp;
 3477     if (index == -1) {
 3478       __ prfm(Address(base, disp), PSTL1KEEP);
 3479     } else {
 3480       Register index_reg = as_Register(index);
 3481       if (disp == 0) {
 3482         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3483       } else {
 3484         __ lea(rscratch1, Address(base, disp));
 3485 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3486       }
 3487     }
 3488   %}
 3489 
 3490   /// mov envcodings
 3491 
 3492   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3493     C2_MacroAssembler _masm(&cbuf);
 3494     uint32_t con = (uint32_t)$src$$constant;
 3495     Register dst_reg = as_Register($dst$$reg);
 3496     if (con == 0) {
 3497       __ movw(dst_reg, zr);
 3498     } else {
 3499       __ movw(dst_reg, con);
 3500     }
 3501   %}
 3502 
 3503   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3504     C2_MacroAssembler _masm(&cbuf);
 3505     Register dst_reg = as_Register($dst$$reg);
 3506     uint64_t con = (uint64_t)$src$$constant;
 3507     if (con == 0) {
 3508       __ mov(dst_reg, zr);
 3509     } else {
 3510       __ mov(dst_reg, con);
 3511     }
 3512   %}
 3513 
 3514   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3515     C2_MacroAssembler _masm(&cbuf);
 3516     Register dst_reg = as_Register($dst$$reg);
 3517     address con = (address)$src$$constant;
 3518     if (con == NULL || con == (address)1) {
 3519       ShouldNotReachHere();
 3520     } else {
 3521       relocInfo::relocType rtype = $src->constant_reloc();
 3522       if (rtype == relocInfo::oop_type) {
 3523         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3524       } else if (rtype == relocInfo::metadata_type) {
 3525         __ mov_metadata(dst_reg, (Metadata*)con);
 3526       } else {
 3527         assert(rtype == relocInfo::none, "unexpected reloc type");
 3528         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3529           __ mov(dst_reg, con);
 3530         } else {
 3531           uint64_t offset;
 3532           __ adrp(dst_reg, con, offset);
 3533           __ add(dst_reg, dst_reg, offset);
 3534         }
 3535       }
 3536     }
 3537   %}
 3538 
 3539   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3540     C2_MacroAssembler _masm(&cbuf);
 3541     Register dst_reg = as_Register($dst$$reg);
 3542     __ mov(dst_reg, zr);
 3543   %}
 3544 
 3545   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3546     C2_MacroAssembler _masm(&cbuf);
 3547     Register dst_reg = as_Register($dst$$reg);
 3548     __ mov(dst_reg, (uint64_t)1);
 3549   %}
 3550 
 3551   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3552     C2_MacroAssembler _masm(&cbuf);
 3553     __ load_byte_map_base($dst$$Register);
 3554   %}
 3555 
 3556   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558     Register dst_reg = as_Register($dst$$reg);
 3559     address con = (address)$src$$constant;
 3560     if (con == NULL) {
 3561       ShouldNotReachHere();
 3562     } else {
 3563       relocInfo::relocType rtype = $src->constant_reloc();
 3564       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3565       __ set_narrow_oop(dst_reg, (jobject)con);
 3566     }
 3567   %}
 3568 
 3569   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3570     C2_MacroAssembler _masm(&cbuf);
 3571     Register dst_reg = as_Register($dst$$reg);
 3572     __ mov(dst_reg, zr);
 3573   %}
 3574 
 3575   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3576     C2_MacroAssembler _masm(&cbuf);
 3577     Register dst_reg = as_Register($dst$$reg);
 3578     address con = (address)$src$$constant;
 3579     if (con == NULL) {
 3580       ShouldNotReachHere();
 3581     } else {
 3582       relocInfo::relocType rtype = $src->constant_reloc();
 3583       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3584       __ set_narrow_klass(dst_reg, (Klass *)con);
 3585     }
 3586   %}
 3587 
 3588   // arithmetic encodings
 3589 
 3590   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3591     C2_MacroAssembler _masm(&cbuf);
 3592     Register dst_reg = as_Register($dst$$reg);
 3593     Register src_reg = as_Register($src1$$reg);
 3594     int32_t con = (int32_t)$src2$$constant;
 3595     // add has primary == 0, subtract has primary == 1
 3596     if ($primary) { con = -con; }
 3597     if (con < 0) {
 3598       __ subw(dst_reg, src_reg, -con);
 3599     } else {
 3600       __ addw(dst_reg, src_reg, con);
 3601     }
 3602   %}
 3603 
 3604   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3605     C2_MacroAssembler _masm(&cbuf);
 3606     Register dst_reg = as_Register($dst$$reg);
 3607     Register src_reg = as_Register($src1$$reg);
 3608     int32_t con = (int32_t)$src2$$constant;
 3609     // add has primary == 0, subtract has primary == 1
 3610     if ($primary) { con = -con; }
 3611     if (con < 0) {
 3612       __ sub(dst_reg, src_reg, -con);
 3613     } else {
 3614       __ add(dst_reg, src_reg, con);
 3615     }
 3616   %}
 3617 
 3618   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3619     C2_MacroAssembler _masm(&cbuf);
 3620    Register dst_reg = as_Register($dst$$reg);
 3621    Register src1_reg = as_Register($src1$$reg);
 3622    Register src2_reg = as_Register($src2$$reg);
 3623     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3627     C2_MacroAssembler _masm(&cbuf);
 3628    Register dst_reg = as_Register($dst$$reg);
 3629    Register src1_reg = as_Register($src1$$reg);
 3630    Register src2_reg = as_Register($src2$$reg);
 3631     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3632   %}
 3633 
 3634   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3635     C2_MacroAssembler _masm(&cbuf);
 3636    Register dst_reg = as_Register($dst$$reg);
 3637    Register src1_reg = as_Register($src1$$reg);
 3638    Register src2_reg = as_Register($src2$$reg);
 3639     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3640   %}
 3641 
 3642   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3643     C2_MacroAssembler _masm(&cbuf);
 3644    Register dst_reg = as_Register($dst$$reg);
 3645    Register src1_reg = as_Register($src1$$reg);
 3646    Register src2_reg = as_Register($src2$$reg);
 3647     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3648   %}
 3649 
 3650   // compare instruction encodings
 3651 
 3652   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3653     C2_MacroAssembler _masm(&cbuf);
 3654     Register reg1 = as_Register($src1$$reg);
 3655     Register reg2 = as_Register($src2$$reg);
 3656     __ cmpw(reg1, reg2);
 3657   %}
 3658 
 3659   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3660     C2_MacroAssembler _masm(&cbuf);
 3661     Register reg = as_Register($src1$$reg);
 3662     int32_t val = $src2$$constant;
 3663     if (val >= 0) {
 3664       __ subsw(zr, reg, val);
 3665     } else {
 3666       __ addsw(zr, reg, -val);
 3667     }
 3668   %}
 3669 
 3670   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3671     C2_MacroAssembler _masm(&cbuf);
 3672     Register reg1 = as_Register($src1$$reg);
 3673     uint32_t val = (uint32_t)$src2$$constant;
 3674     __ movw(rscratch1, val);
 3675     __ cmpw(reg1, rscratch1);
 3676   %}
 3677 
 3678   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3679     C2_MacroAssembler _masm(&cbuf);
 3680     Register reg1 = as_Register($src1$$reg);
 3681     Register reg2 = as_Register($src2$$reg);
 3682     __ cmp(reg1, reg2);
 3683   %}
 3684 
 3685   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3686     C2_MacroAssembler _masm(&cbuf);
 3687     Register reg = as_Register($src1$$reg);
 3688     int64_t val = $src2$$constant;
 3689     if (val >= 0) {
 3690       __ subs(zr, reg, val);
 3691     } else if (val != -val) {
 3692       __ adds(zr, reg, -val);
 3693     } else {
 3694     // aargh, Long.MIN_VALUE is a special case
 3695       __ orr(rscratch1, zr, (uint64_t)val);
 3696       __ subs(zr, reg, rscratch1);
 3697     }
 3698   %}
 3699 
 3700   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3701     C2_MacroAssembler _masm(&cbuf);
 3702     Register reg1 = as_Register($src1$$reg);
 3703     uint64_t val = (uint64_t)$src2$$constant;
 3704     __ mov(rscratch1, val);
 3705     __ cmp(reg1, rscratch1);
 3706   %}
 3707 
 3708   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3709     C2_MacroAssembler _masm(&cbuf);
 3710     Register reg1 = as_Register($src1$$reg);
 3711     Register reg2 = as_Register($src2$$reg);
 3712     __ cmp(reg1, reg2);
 3713   %}
 3714 
 3715   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3716     C2_MacroAssembler _masm(&cbuf);
 3717     Register reg1 = as_Register($src1$$reg);
 3718     Register reg2 = as_Register($src2$$reg);
 3719     __ cmpw(reg1, reg2);
 3720   %}
 3721 
 3722   enc_class aarch64_enc_testp(iRegP src) %{
 3723     C2_MacroAssembler _masm(&cbuf);
 3724     Register reg = as_Register($src$$reg);
 3725     __ cmp(reg, zr);
 3726   %}
 3727 
 3728   enc_class aarch64_enc_testn(iRegN src) %{
 3729     C2_MacroAssembler _masm(&cbuf);
 3730     Register reg = as_Register($src$$reg);
 3731     __ cmpw(reg, zr);
 3732   %}
 3733 
 3734   enc_class aarch64_enc_b(label lbl) %{
 3735     C2_MacroAssembler _masm(&cbuf);
 3736     Label *L = $lbl$$label;
 3737     __ b(*L);
 3738   %}
 3739 
 3740   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3741     C2_MacroAssembler _masm(&cbuf);
 3742     Label *L = $lbl$$label;
 3743     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3744   %}
 3745 
 3746   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3747     C2_MacroAssembler _masm(&cbuf);
 3748     Label *L = $lbl$$label;
 3749     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3750   %}
 3751 
 3752   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3753   %{
 3754      Register sub_reg = as_Register($sub$$reg);
 3755      Register super_reg = as_Register($super$$reg);
 3756      Register temp_reg = as_Register($temp$$reg);
 3757      Register result_reg = as_Register($result$$reg);
 3758 
 3759      Label miss;
 3760      C2_MacroAssembler _masm(&cbuf);
 3761      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3762                                      NULL, &miss,
 3763                                      /*set_cond_codes:*/ true);
 3764      if ($primary) {
 3765        __ mov(result_reg, zr);
 3766      }
 3767      __ bind(miss);
 3768   %}
 3769 
 3770   enc_class aarch64_enc_java_static_call(method meth) %{
 3771     C2_MacroAssembler _masm(&cbuf);
 3772 
 3773     address addr = (address)$meth$$method;
 3774     address call;
 3775     if (!_method) {
 3776       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3777       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3778       if (call == NULL) {
 3779         ciEnv::current()->record_failure("CodeCache is full");
 3780         return;
 3781       }
 3782     } else {
 3783       int method_index = resolved_method_index(cbuf);
 3784       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3785                                                   : static_call_Relocation::spec(method_index);
 3786       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3787       if (call == NULL) {
 3788         ciEnv::current()->record_failure("CodeCache is full");
 3789         return;
 3790       }
 3791       // Emit stub for static call
 3792       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3793       if (stub == NULL) {
 3794         ciEnv::current()->record_failure("CodeCache is full");
 3795         return;
 3796       }
 3797     }
 3798 
 3799     // Only non uncommon_trap calls need to reinitialize ptrue.
 3800     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3801       __ reinitialize_ptrue();
 3802     }
 3803   %}
 3804 
 3805   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3806     C2_MacroAssembler _masm(&cbuf);
 3807     int method_index = resolved_method_index(cbuf);
 3808     address call = __ ic_call((address)$meth$$method, method_index);
 3809     if (call == NULL) {
 3810       ciEnv::current()->record_failure("CodeCache is full");
 3811       return;
 3812     } else if (Compile::current()->max_vector_size() > 0) {
 3813       __ reinitialize_ptrue();
 3814     }
 3815   %}
 3816 
 3817   enc_class aarch64_enc_call_epilog() %{
 3818     C2_MacroAssembler _masm(&cbuf);
 3819     if (VerifyStackAtCalls) {
 3820       // Check that stack depth is unchanged: find majik cookie on stack
 3821       __ call_Unimplemented();
 3822     }
 3823   %}
 3824 
 3825   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3826     C2_MacroAssembler _masm(&cbuf);
 3827 
 3828     // some calls to generated routines (arraycopy code) are scheduled
 3829     // by C2 as runtime calls. if so we can call them using a br (they
 3830     // will be in a reachable segment) otherwise we have to use a blr
 3831     // which loads the absolute address into a register.
 3832     address entry = (address)$meth$$method;
 3833     CodeBlob *cb = CodeCache::find_blob(entry);
 3834     if (cb) {
 3835       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3836       if (call == NULL) {
 3837         ciEnv::current()->record_failure("CodeCache is full");
 3838         return;
 3839       }
 3840     } else {
 3841       Label retaddr;
 3842       __ adr(rscratch2, retaddr);
 3843       __ lea(rscratch1, RuntimeAddress(entry));
 3844       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3845       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3846       __ blr(rscratch1);
 3847       __ bind(retaddr);
 3848       __ add(sp, sp, 2 * wordSize);
 3849     }
 3850     if (Compile::current()->max_vector_size() > 0) {
 3851       __ reinitialize_ptrue();
 3852     }
 3853   %}
 3854 
 3855   enc_class aarch64_enc_rethrow() %{
 3856     C2_MacroAssembler _masm(&cbuf);
 3857     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3858   %}
 3859 
 3860   enc_class aarch64_enc_ret() %{
 3861     C2_MacroAssembler _masm(&cbuf);
 3862 #ifdef ASSERT
 3863     if (Compile::current()->max_vector_size() > 0) {
 3864       __ verify_ptrue();
 3865     }
 3866 #endif
 3867     __ ret(lr);
 3868   %}
 3869 
 3870   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3871     C2_MacroAssembler _masm(&cbuf);
 3872     Register target_reg = as_Register($jump_target$$reg);
 3873     __ br(target_reg);
 3874   %}
 3875 
 3876   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3877     C2_MacroAssembler _masm(&cbuf);
 3878     Register target_reg = as_Register($jump_target$$reg);
 3879     // exception oop should be in r0
 3880     // ret addr has been popped into lr
 3881     // callee expects it in r3
 3882     __ mov(r3, lr);
 3883     __ br(target_reg);
 3884   %}
 3885 
 3886   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3887     C2_MacroAssembler _masm(&cbuf);
 3888     Register oop = as_Register($object$$reg);
 3889     Register box = as_Register($box$$reg);
 3890     Register disp_hdr = as_Register($tmp$$reg);
 3891     Register tmp = as_Register($tmp2$$reg);
 3892     Label cont;
 3893     Label object_has_monitor;
 3894     Label cas_failed;
 3895 
 3896     assert_different_registers(oop, box, tmp, disp_hdr);
 3897 
 3898     // Load markWord from object into displaced_header.
 3899     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3900 
 3901     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3902       __ load_klass(tmp, oop);
 3903       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3904       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3905       __ br(Assembler::NE, cont);
 3906     }
 3907 
 3908     // Check for existing monitor
 3909     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3910 
 3911     if (!UseHeavyMonitors) {
 3912       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3913       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3914 
 3915       // Initialize the box. (Must happen before we update the object mark!)
 3916       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3917 
 3918       // Compare object markWord with an unlocked value (tmp) and if
 3919       // equal exchange the stack address of our box with object markWord.
 3920       // On failure disp_hdr contains the possibly locked markWord.
 3921       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3922                  /*release*/ true, /*weak*/ false, disp_hdr);
 3923       __ br(Assembler::EQ, cont);
 3924 
 3925       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3926 
 3927       // If the compare-and-exchange succeeded, then we found an unlocked
 3928       // object, will have now locked it will continue at label cont
 3929 
 3930       __ bind(cas_failed);
 3931       // We did not see an unlocked object so try the fast recursive case.
 3932 
 3933       // Check if the owner is self by comparing the value in the
 3934       // markWord of object (disp_hdr) with the stack pointer.
 3935       __ mov(rscratch1, sp);
 3936       __ sub(disp_hdr, disp_hdr, rscratch1);
 3937       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3938       // If condition is true we are cont and hence we can store 0 as the
 3939       // displaced header in the box, which indicates that it is a recursive lock.
 3940       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3941       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3942     } else {
 3943       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3944     }
 3945     __ b(cont);
 3946 
 3947     // Handle existing monitor.
 3948     __ bind(object_has_monitor);
 3949 
 3950     // The object's monitor m is unlocked iff m->owner == NULL,
 3951     // otherwise m->owner may contain a thread or a stack address.
 3952     //
 3953     // Try to CAS m->owner from NULL to current thread.
 3954     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3955     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3956                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3957 
 3958     // Store a non-null value into the box to avoid looking like a re-entrant
 3959     // lock. The fast-path monitor unlock code checks for
 3960     // markWord::monitor_value so use markWord::unused_mark which has the
 3961     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3962     __ mov(tmp, (address)markWord::unused_mark().value());
 3963     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3964 
 3965     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3966 
 3967     __ cmp(rscratch1, rthread);
 3968     __ br(Assembler::NE, cont); // Check for recursive locking
 3969 
 3970     // Recursive lock case
 3971     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3972     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3973 
 3974     __ bind(cont);
 3975     // flag == EQ indicates success
 3976     // flag == NE indicates failure
 3977   %}
 3978 
 3979   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3980     C2_MacroAssembler _masm(&cbuf);
 3981     Register oop = as_Register($object$$reg);
 3982     Register box = as_Register($box$$reg);
 3983     Register disp_hdr = as_Register($tmp$$reg);
 3984     Register tmp = as_Register($tmp2$$reg);
 3985     Label cont;
 3986     Label object_has_monitor;
 3987 
 3988     assert_different_registers(oop, box, tmp, disp_hdr);
 3989 
 3990     if (!UseHeavyMonitors) {
 3991       // Find the lock address and load the displaced header from the stack.
 3992       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3993 
 3994       // If the displaced header is 0, we have a recursive unlock.
 3995       __ cmp(disp_hdr, zr);
 3996       __ br(Assembler::EQ, cont);
 3997     }
 3998 
 3999     // Handle existing monitor.
 4000     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 4001     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 4002 
 4003     if (!UseHeavyMonitors) {
 4004       // Check if it is still a light weight lock, this is is true if we
 4005       // see the stack address of the basicLock in the markWord of the
 4006       // object.
 4007 
 4008       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 4009                  /*release*/ true, /*weak*/ false, tmp);
 4010     } else {
 4011       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 4012     }
 4013     __ b(cont);
 4014 
 4015     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 4016 
 4017     // Handle existing monitor.
 4018     __ bind(object_has_monitor);
 4019     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 4020     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 4021     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4022 
 4023     Label notRecursive;
 4024     __ cbz(disp_hdr, notRecursive);
 4025 
 4026     // Recursive lock
 4027     __ sub(disp_hdr, disp_hdr, 1u);
 4028     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 4029     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 4030     __ b(cont);
 4031 
 4032     __ bind(notRecursive);
 4033     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 4034     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 4035     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 4036     __ cmp(rscratch1, zr); // Sets flags for result
 4037     __ cbnz(rscratch1, cont);
 4038     // need a release store here
 4039     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 4040     __ stlr(zr, tmp); // set unowned
 4041 
 4042     __ bind(cont);
 4043     // flag == EQ indicates success
 4044     // flag == NE indicates failure
 4045   %}
 4046 
 4047 %}
 4048 
 4049 //----------FRAME--------------------------------------------------------------
 4050 // Definition of frame structure and management information.
 4051 //
 4052 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4053 //                             |   (to get allocators register number
 4054 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4055 //  r   CALLER     |        |
 4056 //  o     |        +--------+      pad to even-align allocators stack-slot
 4057 //  w     V        |  pad0  |        numbers; owned by CALLER
 4058 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4059 //  h     ^        |   in   |  5
 4060 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4061 //  |     |        |        |  3
 4062 //  |     |        +--------+
 4063 //  V     |        | old out|      Empty on Intel, window on Sparc
 4064 //        |    old |preserve|      Must be even aligned.
 4065 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4066 //        |        |   in   |  3   area for Intel ret address
 4067 //     Owned by    |preserve|      Empty on Sparc.
 4068 //       SELF      +--------+
 4069 //        |        |  pad2  |  2   pad to align old SP
 4070 //        |        +--------+  1
 4071 //        |        | locks  |  0
 4072 //        |        +--------+----> OptoReg::stack0(), even aligned
 4073 //        |        |  pad1  | 11   pad to align new SP
 4074 //        |        +--------+
 4075 //        |        |        | 10
 4076 //        |        | spills |  9   spills
 4077 //        V        |        |  8   (pad0 slot for callee)
 4078 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4079 //        ^        |  out   |  7
 4080 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4081 //     Owned by    +--------+
 4082 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4083 //        |    new |preserve|      Must be even-aligned.
 4084 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4085 //        |        |        |
 4086 //
 4087 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4088 //         known from SELF's arguments and the Java calling convention.
 4089 //         Region 6-7 is determined per call site.
 4090 // Note 2: If the calling convention leaves holes in the incoming argument
 4091 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4092 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4093 //         incoming area, as the Java calling convention is completely under
 4094 //         the control of the AD file.  Doubles can be sorted and packed to
 4095 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4096 //         varargs C calling conventions.
 4097 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4098 //         even aligned with pad0 as needed.
 4099 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4100 //           (the latter is true on Intel but is it false on AArch64?)
 4101 //         region 6-11 is even aligned; it may be padded out more so that
 4102 //         the region from SP to FP meets the minimum stack alignment.
 4103 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4104 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4105 //         SP meets the minimum alignment.
 4106 
 4107 frame %{
 4108   // These three registers define part of the calling convention
 4109   // between compiled code and the interpreter.
 4110 
 4111   // Inline Cache Register or Method for I2C.
 4112   inline_cache_reg(R12);
 4113 
 4114   // Number of stack slots consumed by locking an object
 4115   sync_stack_slots(2);
 4116 
 4117   // Compiled code's Frame Pointer
 4118   frame_pointer(R31);
 4119 
 4120   // Interpreter stores its frame pointer in a register which is
 4121   // stored to the stack by I2CAdaptors.
 4122   // I2CAdaptors convert from interpreted java to compiled java.
 4123   interpreter_frame_pointer(R29);
 4124 
 4125   // Stack alignment requirement
 4126   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4127 
 4128   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4129   // for calls to C.  Supports the var-args backing area for register parms.
 4130   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4131 
 4132   // The after-PROLOG location of the return address.  Location of
 4133   // return address specifies a type (REG or STACK) and a number
 4134   // representing the register number (i.e. - use a register name) or
 4135   // stack slot.
 4136   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4137   // Otherwise, it is above the locks and verification slot and alignment word
 4138   // TODO this may well be correct but need to check why that - 2 is there
 4139   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4140   // which folds in the space used for monitors
 4141   return_addr(STACK - 2 +
 4142               align_up((Compile::current()->in_preserve_stack_slots() +
 4143                         Compile::current()->fixed_slots()),
 4144                        stack_alignment_in_slots()));
 4145 
 4146   // Location of compiled Java return values.  Same as C for now.
 4147   return_value
 4148   %{
 4149     // TODO do we allow ideal_reg == Op_RegN???
 4150     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4151            "only return normal values");
 4152 
 4153     static const int lo[Op_RegL + 1] = { // enum name
 4154       0,                                 // Op_Node
 4155       0,                                 // Op_Set
 4156       R0_num,                            // Op_RegN
 4157       R0_num,                            // Op_RegI
 4158       R0_num,                            // Op_RegP
 4159       V0_num,                            // Op_RegF
 4160       V0_num,                            // Op_RegD
 4161       R0_num                             // Op_RegL
 4162     };
 4163 
 4164     static const int hi[Op_RegL + 1] = { // enum name
 4165       0,                                 // Op_Node
 4166       0,                                 // Op_Set
 4167       OptoReg::Bad,                      // Op_RegN
 4168       OptoReg::Bad,                      // Op_RegI
 4169       R0_H_num,                          // Op_RegP
 4170       OptoReg::Bad,                      // Op_RegF
 4171       V0_H_num,                          // Op_RegD
 4172       R0_H_num                           // Op_RegL
 4173     };
 4174 
 4175     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4176   %}
 4177 %}
 4178 
 4179 //----------ATTRIBUTES---------------------------------------------------------
 4180 //----------Operand Attributes-------------------------------------------------
 4181 op_attrib op_cost(1);        // Required cost attribute
 4182 
 4183 //----------Instruction Attributes---------------------------------------------
 4184 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4185 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4186 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4187                                 // a non-matching short branch variant
 4188                                 // of some long branch?
 4189 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4190                                 // be a power of 2) specifies the
 4191                                 // alignment that some part of the
 4192                                 // instruction (not necessarily the
 4193                                 // start) requires.  If > 1, a
 4194                                 // compute_padding() function must be
 4195                                 // provided for the instruction
 4196 
 4197 //----------OPERANDS-----------------------------------------------------------
 4198 // Operand definitions must precede instruction definitions for correct parsing
 4199 // in the ADLC because operands constitute user defined types which are used in
 4200 // instruction definitions.
 4201 
 4202 //----------Simple Operands----------------------------------------------------
 4203 
 4204 // Integer operands 32 bit
 4205 // 32 bit immediate
 4206 operand immI()
 4207 %{
 4208   match(ConI);
 4209 
 4210   op_cost(0);
 4211   format %{ %}
 4212   interface(CONST_INTER);
 4213 %}
 4214 
 4215 // 32 bit zero
 4216 operand immI0()
 4217 %{
 4218   predicate(n->get_int() == 0);
 4219   match(ConI);
 4220 
 4221   op_cost(0);
 4222   format %{ %}
 4223   interface(CONST_INTER);
 4224 %}
 4225 
 4226 // 32 bit unit increment
 4227 operand immI_1()
 4228 %{
 4229   predicate(n->get_int() == 1);
 4230   match(ConI);
 4231 
 4232   op_cost(0);
 4233   format %{ %}
 4234   interface(CONST_INTER);
 4235 %}
 4236 
 4237 // 32 bit unit decrement
 4238 operand immI_M1()
 4239 %{
 4240   predicate(n->get_int() == -1);
 4241   match(ConI);
 4242 
 4243   op_cost(0);
 4244   format %{ %}
 4245   interface(CONST_INTER);
 4246 %}
 4247 
 4248 // Shift values for add/sub extension shift
 4249 operand immIExt()
 4250 %{
 4251   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4252   match(ConI);
 4253 
 4254   op_cost(0);
 4255   format %{ %}
 4256   interface(CONST_INTER);
 4257 %}
 4258 
 4259 operand immI_gt_1()
 4260 %{
 4261   predicate(n->get_int() > 1);
 4262   match(ConI);
 4263 
 4264   op_cost(0);
 4265   format %{ %}
 4266   interface(CONST_INTER);
 4267 %}
 4268 
 4269 operand immI_le_4()
 4270 %{
 4271   predicate(n->get_int() <= 4);
 4272   match(ConI);
 4273 
 4274   op_cost(0);
 4275   format %{ %}
 4276   interface(CONST_INTER);
 4277 %}
 4278 
 4279 operand immI_31()
 4280 %{
 4281   predicate(n->get_int() == 31);
 4282   match(ConI);
 4283 
 4284   op_cost(0);
 4285   format %{ %}
 4286   interface(CONST_INTER);
 4287 %}
 4288 
 4289 operand immI_2()
 4290 %{
 4291   predicate(n->get_int() == 2);
 4292   match(ConI);
 4293 
 4294   op_cost(0);
 4295   format %{ %}
 4296   interface(CONST_INTER);
 4297 %}
 4298 
 4299 operand immI_4()
 4300 %{
 4301   predicate(n->get_int() == 4);
 4302   match(ConI);
 4303 
 4304   op_cost(0);
 4305   format %{ %}
 4306   interface(CONST_INTER);
 4307 %}
 4308 
 4309 operand immI_8()
 4310 %{
 4311   predicate(n->get_int() == 8);
 4312   match(ConI);
 4313 
 4314   op_cost(0);
 4315   format %{ %}
 4316   interface(CONST_INTER);
 4317 %}
 4318 
 4319 operand immI_16()
 4320 %{
 4321   predicate(n->get_int() == 16);
 4322   match(ConI);
 4323 
 4324   op_cost(0);
 4325   format %{ %}
 4326   interface(CONST_INTER);
 4327 %}
 4328 
 4329 operand immI_24()
 4330 %{
 4331   predicate(n->get_int() == 24);
 4332   match(ConI);
 4333 
 4334   op_cost(0);
 4335   format %{ %}
 4336   interface(CONST_INTER);
 4337 %}
 4338 
 4339 operand immI_32()
 4340 %{
 4341   predicate(n->get_int() == 32);
 4342   match(ConI);
 4343 
 4344   op_cost(0);
 4345   format %{ %}
 4346   interface(CONST_INTER);
 4347 %}
 4348 
 4349 operand immI_48()
 4350 %{
 4351   predicate(n->get_int() == 48);
 4352   match(ConI);
 4353 
 4354   op_cost(0);
 4355   format %{ %}
 4356   interface(CONST_INTER);
 4357 %}
 4358 
 4359 operand immI_56()
 4360 %{
 4361   predicate(n->get_int() == 56);
 4362   match(ConI);
 4363 
 4364   op_cost(0);
 4365   format %{ %}
 4366   interface(CONST_INTER);
 4367 %}
 4368 
 4369 operand immI_63()
 4370 %{
 4371   predicate(n->get_int() == 63);
 4372   match(ConI);
 4373 
 4374   op_cost(0);
 4375   format %{ %}
 4376   interface(CONST_INTER);
 4377 %}
 4378 
 4379 operand immI_64()
 4380 %{
 4381   predicate(n->get_int() == 64);
 4382   match(ConI);
 4383 
 4384   op_cost(0);
 4385   format %{ %}
 4386   interface(CONST_INTER);
 4387 %}
 4388 
 4389 operand immI_255()
 4390 %{
 4391   predicate(n->get_int() == 255);
 4392   match(ConI);
 4393 
 4394   op_cost(0);
 4395   format %{ %}
 4396   interface(CONST_INTER);
 4397 %}
 4398 
 4399 operand immI_65535()
 4400 %{
 4401   predicate(n->get_int() == 65535);
 4402   match(ConI);
 4403 
 4404   op_cost(0);
 4405   format %{ %}
 4406   interface(CONST_INTER);
 4407 %}
 4408 
 4409 operand immL_255()
 4410 %{
 4411   predicate(n->get_long() == 255L);
 4412   match(ConL);
 4413 
 4414   op_cost(0);
 4415   format %{ %}
 4416   interface(CONST_INTER);
 4417 %}
 4418 
 4419 operand immL_65535()
 4420 %{
 4421   predicate(n->get_long() == 65535L);
 4422   match(ConL);
 4423 
 4424   op_cost(0);
 4425   format %{ %}
 4426   interface(CONST_INTER);
 4427 %}
 4428 
 4429 operand immL_4294967295()
 4430 %{
 4431   predicate(n->get_long() == 4294967295L);
 4432   match(ConL);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 operand immL_bitmask()
 4440 %{
 4441   predicate((n->get_long() != 0)
 4442             && ((n->get_long() & 0xc000000000000000l) == 0)
 4443             && is_power_of_2(n->get_long() + 1));
 4444   match(ConL);
 4445 
 4446   op_cost(0);
 4447   format %{ %}
 4448   interface(CONST_INTER);
 4449 %}
 4450 
 4451 operand immI_bitmask()
 4452 %{
 4453   predicate((n->get_int() != 0)
 4454             && ((n->get_int() & 0xc0000000) == 0)
 4455             && is_power_of_2(n->get_int() + 1));
 4456   match(ConI);
 4457 
 4458   op_cost(0);
 4459   format %{ %}
 4460   interface(CONST_INTER);
 4461 %}
 4462 
 4463 operand immL_positive_bitmaskI()
 4464 %{
 4465   predicate((n->get_long() != 0)
 4466             && ((julong)n->get_long() < 0x80000000ULL)
 4467             && is_power_of_2(n->get_long() + 1));
 4468   match(ConL);
 4469 
 4470   op_cost(0);
 4471   format %{ %}
 4472   interface(CONST_INTER);
 4473 %}
 4474 
 4475 // Scale values for scaled offset addressing modes (up to long but not quad)
 4476 operand immIScale()
 4477 %{
 4478   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4479   match(ConI);
 4480 
 4481   op_cost(0);
 4482   format %{ %}
 4483   interface(CONST_INTER);
 4484 %}
 4485 
 4486 // 26 bit signed offset -- for pc-relative branches
 4487 operand immI26()
 4488 %{
 4489   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4490   match(ConI);
 4491 
 4492   op_cost(0);
 4493   format %{ %}
 4494   interface(CONST_INTER);
 4495 %}
 4496 
 4497 // 19 bit signed offset -- for pc-relative loads
 4498 operand immI19()
 4499 %{
 4500   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4501   match(ConI);
 4502 
 4503   op_cost(0);
 4504   format %{ %}
 4505   interface(CONST_INTER);
 4506 %}
 4507 
 4508 // 12 bit unsigned offset -- for base plus immediate loads
 4509 operand immIU12()
 4510 %{
 4511   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4512   match(ConI);
 4513 
 4514   op_cost(0);
 4515   format %{ %}
 4516   interface(CONST_INTER);
 4517 %}
 4518 
 4519 operand immLU12()
 4520 %{
 4521   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4522   match(ConL);
 4523 
 4524   op_cost(0);
 4525   format %{ %}
 4526   interface(CONST_INTER);
 4527 %}
 4528 
 4529 // Offset for scaled or unscaled immediate loads and stores
 4530 operand immIOffset()
 4531 %{
 4532   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4533   match(ConI);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 operand immIOffset1()
 4541 %{
 4542   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4543   match(ConI);
 4544 
 4545   op_cost(0);
 4546   format %{ %}
 4547   interface(CONST_INTER);
 4548 %}
 4549 
 4550 operand immIOffset2()
 4551 %{
 4552   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4553   match(ConI);
 4554 
 4555   op_cost(0);
 4556   format %{ %}
 4557   interface(CONST_INTER);
 4558 %}
 4559 
 4560 operand immIOffset4()
 4561 %{
 4562   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4563   match(ConI);
 4564 
 4565   op_cost(0);
 4566   format %{ %}
 4567   interface(CONST_INTER);
 4568 %}
 4569 
 4570 operand immIOffset8()
 4571 %{
 4572   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4573   match(ConI);
 4574 
 4575   op_cost(0);
 4576   format %{ %}
 4577   interface(CONST_INTER);
 4578 %}
 4579 
 4580 operand immIOffset16()
 4581 %{
 4582   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4583   match(ConI);
 4584 
 4585   op_cost(0);
 4586   format %{ %}
 4587   interface(CONST_INTER);
 4588 %}
 4589 
 4590 operand immLoffset()
 4591 %{
 4592   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4593   match(ConL);
 4594 
 4595   op_cost(0);
 4596   format %{ %}
 4597   interface(CONST_INTER);
 4598 %}
 4599 
 4600 operand immLoffset1()
 4601 %{
 4602   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4603   match(ConL);
 4604 
 4605   op_cost(0);
 4606   format %{ %}
 4607   interface(CONST_INTER);
 4608 %}
 4609 
 4610 operand immLoffset2()
 4611 %{
 4612   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4613   match(ConL);
 4614 
 4615   op_cost(0);
 4616   format %{ %}
 4617   interface(CONST_INTER);
 4618 %}
 4619 
 4620 operand immLoffset4()
 4621 %{
 4622   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4623   match(ConL);
 4624 
 4625   op_cost(0);
 4626   format %{ %}
 4627   interface(CONST_INTER);
 4628 %}
 4629 
 4630 operand immLoffset8()
 4631 %{
 4632   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4633   match(ConL);
 4634 
 4635   op_cost(0);
 4636   format %{ %}
 4637   interface(CONST_INTER);
 4638 %}
 4639 
 4640 operand immLoffset16()
 4641 %{
 4642   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4643   match(ConL);
 4644 
 4645   op_cost(0);
 4646   format %{ %}
 4647   interface(CONST_INTER);
 4648 %}
 4649 
 4650 // 8 bit signed value.
 4651 operand immI8()
 4652 %{
 4653   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4654   match(ConI);
 4655 
 4656   op_cost(0);
 4657   format %{ %}
 4658   interface(CONST_INTER);
 4659 %}
 4660 
 4661 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4662 operand immI8_shift8()
 4663 %{
 4664   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4665             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4666   match(ConI);
 4667 
 4668   op_cost(0);
 4669   format %{ %}
 4670   interface(CONST_INTER);
 4671 %}
 4672 
 4673 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4674 operand immL8_shift8()
 4675 %{
 4676   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4677             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4678   match(ConL);
 4679 
 4680   op_cost(0);
 4681   format %{ %}
 4682   interface(CONST_INTER);
 4683 %}
 4684 
 4685 // 8 bit integer valid for vector add sub immediate
 4686 operand immBAddSubV()
 4687 %{
 4688   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4689   match(ConI);
 4690 
 4691   op_cost(0);
 4692   format %{ %}
 4693   interface(CONST_INTER);
 4694 %}
 4695 
 4696 // 32 bit integer valid for add sub immediate
 4697 operand immIAddSub()
 4698 %{
 4699   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4700   match(ConI);
 4701   op_cost(0);
 4702   format %{ %}
 4703   interface(CONST_INTER);
 4704 %}
 4705 
 4706 // 32 bit integer valid for vector add sub immediate
 4707 operand immIAddSubV()
 4708 %{
 4709   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4710   match(ConI);
 4711 
 4712   op_cost(0);
 4713   format %{ %}
 4714   interface(CONST_INTER);
 4715 %}
 4716 
 4717 // 32 bit unsigned integer valid for logical immediate
 4718 
 4719 operand immBLog()
 4720 %{
 4721   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4722   match(ConI);
 4723 
 4724   op_cost(0);
 4725   format %{ %}
 4726   interface(CONST_INTER);
 4727 %}
 4728 
 4729 operand immSLog()
 4730 %{
 4731   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4732   match(ConI);
 4733 
 4734   op_cost(0);
 4735   format %{ %}
 4736   interface(CONST_INTER);
 4737 %}
 4738 
 4739 operand immILog()
 4740 %{
 4741   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4742   match(ConI);
 4743 
 4744   op_cost(0);
 4745   format %{ %}
 4746   interface(CONST_INTER);
 4747 %}
 4748 
 4749 // Integer operands 64 bit
 4750 // 64 bit immediate
 4751 operand immL()
 4752 %{
 4753   match(ConL);
 4754 
 4755   op_cost(0);
 4756   format %{ %}
 4757   interface(CONST_INTER);
 4758 %}
 4759 
 4760 // 64 bit zero
 4761 operand immL0()
 4762 %{
 4763   predicate(n->get_long() == 0);
 4764   match(ConL);
 4765 
 4766   op_cost(0);
 4767   format %{ %}
 4768   interface(CONST_INTER);
 4769 %}
 4770 
 4771 // 64 bit unit increment
 4772 operand immL_1()
 4773 %{
 4774   predicate(n->get_long() == 1);
 4775   match(ConL);
 4776 
 4777   op_cost(0);
 4778   format %{ %}
 4779   interface(CONST_INTER);
 4780 %}
 4781 
 4782 // 64 bit unit decrement
 4783 operand immL_M1()
 4784 %{
 4785   predicate(n->get_long() == -1);
 4786   match(ConL);
 4787 
 4788   op_cost(0);
 4789   format %{ %}
 4790   interface(CONST_INTER);
 4791 %}
 4792 
 4793 // 32 bit offset of pc in thread anchor
 4794 
 4795 operand immL_pc_off()
 4796 %{
 4797   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4798                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4799   match(ConL);
 4800 
 4801   op_cost(0);
 4802   format %{ %}
 4803   interface(CONST_INTER);
 4804 %}
 4805 
 4806 // 64 bit integer valid for add sub immediate
 4807 operand immLAddSub()
 4808 %{
 4809   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4810   match(ConL);
 4811   op_cost(0);
 4812   format %{ %}
 4813   interface(CONST_INTER);
 4814 %}
 4815 
 4816 // 64 bit integer valid for addv subv immediate
 4817 operand immLAddSubV()
 4818 %{
 4819   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4820   match(ConL);
 4821 
 4822   op_cost(0);
 4823   format %{ %}
 4824   interface(CONST_INTER);
 4825 %}
 4826 
 4827 // 64 bit integer valid for logical immediate
 4828 operand immLLog()
 4829 %{
 4830   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4831   match(ConL);
 4832   op_cost(0);
 4833   format %{ %}
 4834   interface(CONST_INTER);
 4835 %}
 4836 
 4837 // Long Immediate: low 32-bit mask
 4838 operand immL_32bits()
 4839 %{
 4840   predicate(n->get_long() == 0xFFFFFFFFL);
 4841   match(ConL);
 4842   op_cost(0);
 4843   format %{ %}
 4844   interface(CONST_INTER);
 4845 %}
 4846 
 4847 // Pointer operands
 4848 // Pointer Immediate
 4849 operand immP()
 4850 %{
 4851   match(ConP);
 4852 
 4853   op_cost(0);
 4854   format %{ %}
 4855   interface(CONST_INTER);
 4856 %}
 4857 
 4858 // NULL Pointer Immediate
 4859 operand immP0()
 4860 %{
 4861   predicate(n->get_ptr() == 0);
 4862   match(ConP);
 4863 
 4864   op_cost(0);
 4865   format %{ %}
 4866   interface(CONST_INTER);
 4867 %}
 4868 
 4869 // Pointer Immediate One
 4870 // this is used in object initialization (initial object header)
 4871 operand immP_1()
 4872 %{
 4873   predicate(n->get_ptr() == 1);
 4874   match(ConP);
 4875 
 4876   op_cost(0);
 4877   format %{ %}
 4878   interface(CONST_INTER);
 4879 %}
 4880 
 4881 // Card Table Byte Map Base
 4882 operand immByteMapBase()
 4883 %{
 4884   // Get base of card map
 4885   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4886             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4887   match(ConP);
 4888 
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(CONST_INTER);
 4892 %}
 4893 
 4894 // Pointer Immediate Minus One
 4895 // this is used when we want to write the current PC to the thread anchor
 4896 operand immP_M1()
 4897 %{
 4898   predicate(n->get_ptr() == -1);
 4899   match(ConP);
 4900 
 4901   op_cost(0);
 4902   format %{ %}
 4903   interface(CONST_INTER);
 4904 %}
 4905 
 4906 // Pointer Immediate Minus Two
 4907 // this is used when we want to write the current PC to the thread anchor
 4908 operand immP_M2()
 4909 %{
 4910   predicate(n->get_ptr() == -2);
 4911   match(ConP);
 4912 
 4913   op_cost(0);
 4914   format %{ %}
 4915   interface(CONST_INTER);
 4916 %}
 4917 
 4918 // Float and Double operands
 4919 // Double Immediate
 4920 operand immD()
 4921 %{
 4922   match(ConD);
 4923   op_cost(0);
 4924   format %{ %}
 4925   interface(CONST_INTER);
 4926 %}
 4927 
 4928 // Double Immediate: +0.0d
 4929 operand immD0()
 4930 %{
 4931   predicate(jlong_cast(n->getd()) == 0);
 4932   match(ConD);
 4933 
 4934   op_cost(0);
 4935   format %{ %}
 4936   interface(CONST_INTER);
 4937 %}
 4938 
 4939 // constant 'double +0.0'.
 4940 operand immDPacked()
 4941 %{
 4942   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4943   match(ConD);
 4944   op_cost(0);
 4945   format %{ %}
 4946   interface(CONST_INTER);
 4947 %}
 4948 
 4949 // Float Immediate
 4950 operand immF()
 4951 %{
 4952   match(ConF);
 4953   op_cost(0);
 4954   format %{ %}
 4955   interface(CONST_INTER);
 4956 %}
 4957 
 4958 // Float Immediate: +0.0f.
 4959 operand immF0()
 4960 %{
 4961   predicate(jint_cast(n->getf()) == 0);
 4962   match(ConF);
 4963 
 4964   op_cost(0);
 4965   format %{ %}
 4966   interface(CONST_INTER);
 4967 %}
 4968 
 4969 //
 4970 operand immFPacked()
 4971 %{
 4972   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4973   match(ConF);
 4974   op_cost(0);
 4975   format %{ %}
 4976   interface(CONST_INTER);
 4977 %}
 4978 
 4979 // Narrow pointer operands
 4980 // Narrow Pointer Immediate
 4981 operand immN()
 4982 %{
 4983   match(ConN);
 4984 
 4985   op_cost(0);
 4986   format %{ %}
 4987   interface(CONST_INTER);
 4988 %}
 4989 
 4990 // Narrow NULL Pointer Immediate
 4991 operand immN0()
 4992 %{
 4993   predicate(n->get_narrowcon() == 0);
 4994   match(ConN);
 4995 
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(CONST_INTER);
 4999 %}
 5000 
 5001 operand immNKlass()
 5002 %{
 5003   match(ConNKlass);
 5004 
 5005   op_cost(0);
 5006   format %{ %}
 5007   interface(CONST_INTER);
 5008 %}
 5009 
 5010 // Integer 32 bit Register Operands
 5011 // Integer 32 bitRegister (excludes SP)
 5012 operand iRegI()
 5013 %{
 5014   constraint(ALLOC_IN_RC(any_reg32));
 5015   match(RegI);
 5016   match(iRegINoSp);
 5017   op_cost(0);
 5018   format %{ %}
 5019   interface(REG_INTER);
 5020 %}
 5021 
 5022 // Integer 32 bit Register not Special
 5023 operand iRegINoSp()
 5024 %{
 5025   constraint(ALLOC_IN_RC(no_special_reg32));
 5026   match(RegI);
 5027   op_cost(0);
 5028   format %{ %}
 5029   interface(REG_INTER);
 5030 %}
 5031 
 5032 // Integer 64 bit Register Operands
 5033 // Integer 64 bit Register (includes SP)
 5034 operand iRegL()
 5035 %{
 5036   constraint(ALLOC_IN_RC(any_reg));
 5037   match(RegL);
 5038   match(iRegLNoSp);
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(REG_INTER);
 5042 %}
 5043 
 5044 // Integer 64 bit Register not Special
 5045 operand iRegLNoSp()
 5046 %{
 5047   constraint(ALLOC_IN_RC(no_special_reg));
 5048   match(RegL);
 5049   match(iRegL_R0);
 5050   format %{ %}
 5051   interface(REG_INTER);
 5052 %}
 5053 
 5054 // Pointer Register Operands
 5055 // Pointer Register
 5056 operand iRegP()
 5057 %{
 5058   constraint(ALLOC_IN_RC(ptr_reg));
 5059   match(RegP);
 5060   match(iRegPNoSp);
 5061   match(iRegP_R0);
 5062   //match(iRegP_R2);
 5063   //match(iRegP_R4);
 5064   //match(iRegP_R5);
 5065   match(thread_RegP);
 5066   op_cost(0);
 5067   format %{ %}
 5068   interface(REG_INTER);
 5069 %}
 5070 
 5071 // Pointer 64 bit Register not Special
 5072 operand iRegPNoSp()
 5073 %{
 5074   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5075   match(RegP);
 5076   // match(iRegP);
 5077   // match(iRegP_R0);
 5078   // match(iRegP_R2);
 5079   // match(iRegP_R4);
 5080   // match(iRegP_R5);
 5081   // match(thread_RegP);
 5082   op_cost(0);
 5083   format %{ %}
 5084   interface(REG_INTER);
 5085 %}
 5086 
 5087 // Pointer 64 bit Register R0 only
 5088 operand iRegP_R0()
 5089 %{
 5090   constraint(ALLOC_IN_RC(r0_reg));
 5091   match(RegP);
 5092   // match(iRegP);
 5093   match(iRegPNoSp);
 5094   op_cost(0);
 5095   format %{ %}
 5096   interface(REG_INTER);
 5097 %}
 5098 
 5099 // Pointer 64 bit Register R1 only
 5100 operand iRegP_R1()
 5101 %{
 5102   constraint(ALLOC_IN_RC(r1_reg));
 5103   match(RegP);
 5104   // match(iRegP);
 5105   match(iRegPNoSp);
 5106   op_cost(0);
 5107   format %{ %}
 5108   interface(REG_INTER);
 5109 %}
 5110 
 5111 // Pointer 64 bit Register R2 only
 5112 operand iRegP_R2()
 5113 %{
 5114   constraint(ALLOC_IN_RC(r2_reg));
 5115   match(RegP);
 5116   // match(iRegP);
 5117   match(iRegPNoSp);
 5118   op_cost(0);
 5119   format %{ %}
 5120   interface(REG_INTER);
 5121 %}
 5122 
 5123 // Pointer 64 bit Register R3 only
 5124 operand iRegP_R3()
 5125 %{
 5126   constraint(ALLOC_IN_RC(r3_reg));
 5127   match(RegP);
 5128   // match(iRegP);
 5129   match(iRegPNoSp);
 5130   op_cost(0);
 5131   format %{ %}
 5132   interface(REG_INTER);
 5133 %}
 5134 
 5135 // Pointer 64 bit Register R4 only
 5136 operand iRegP_R4()
 5137 %{
 5138   constraint(ALLOC_IN_RC(r4_reg));
 5139   match(RegP);
 5140   // match(iRegP);
 5141   match(iRegPNoSp);
 5142   op_cost(0);
 5143   format %{ %}
 5144   interface(REG_INTER);
 5145 %}
 5146 
 5147 // Pointer 64 bit Register R5 only
 5148 operand iRegP_R5()
 5149 %{
 5150   constraint(ALLOC_IN_RC(r5_reg));
 5151   match(RegP);
 5152   // match(iRegP);
 5153   match(iRegPNoSp);
 5154   op_cost(0);
 5155   format %{ %}
 5156   interface(REG_INTER);
 5157 %}
 5158 
 5159 // Pointer 64 bit Register R10 only
 5160 operand iRegP_R10()
 5161 %{
 5162   constraint(ALLOC_IN_RC(r10_reg));
 5163   match(RegP);
 5164   // match(iRegP);
 5165   match(iRegPNoSp);
 5166   op_cost(0);
 5167   format %{ %}
 5168   interface(REG_INTER);
 5169 %}
 5170 
 5171 // Long 64 bit Register R0 only
 5172 operand iRegL_R0()
 5173 %{
 5174   constraint(ALLOC_IN_RC(r0_reg));
 5175   match(RegL);
 5176   match(iRegLNoSp);
 5177   op_cost(0);
 5178   format %{ %}
 5179   interface(REG_INTER);
 5180 %}
 5181 
 5182 // Long 64 bit Register R2 only
 5183 operand iRegL_R2()
 5184 %{
 5185   constraint(ALLOC_IN_RC(r2_reg));
 5186   match(RegL);
 5187   match(iRegLNoSp);
 5188   op_cost(0);
 5189   format %{ %}
 5190   interface(REG_INTER);
 5191 %}
 5192 
 5193 // Long 64 bit Register R3 only
 5194 operand iRegL_R3()
 5195 %{
 5196   constraint(ALLOC_IN_RC(r3_reg));
 5197   match(RegL);
 5198   match(iRegLNoSp);
 5199   op_cost(0);
 5200   format %{ %}
 5201   interface(REG_INTER);
 5202 %}
 5203 
 5204 // Long 64 bit Register R11 only
 5205 operand iRegL_R11()
 5206 %{
 5207   constraint(ALLOC_IN_RC(r11_reg));
 5208   match(RegL);
 5209   match(iRegLNoSp);
 5210   op_cost(0);
 5211   format %{ %}
 5212   interface(REG_INTER);
 5213 %}
 5214 
 5215 // Pointer 64 bit Register FP only
 5216 operand iRegP_FP()
 5217 %{
 5218   constraint(ALLOC_IN_RC(fp_reg));
 5219   match(RegP);
 5220   // match(iRegP);
 5221   op_cost(0);
 5222   format %{ %}
 5223   interface(REG_INTER);
 5224 %}
 5225 
 5226 // Register R0 only
 5227 operand iRegI_R0()
 5228 %{
 5229   constraint(ALLOC_IN_RC(int_r0_reg));
 5230   match(RegI);
 5231   match(iRegINoSp);
 5232   op_cost(0);
 5233   format %{ %}
 5234   interface(REG_INTER);
 5235 %}
 5236 
 5237 // Register R2 only
 5238 operand iRegI_R2()
 5239 %{
 5240   constraint(ALLOC_IN_RC(int_r2_reg));
 5241   match(RegI);
 5242   match(iRegINoSp);
 5243   op_cost(0);
 5244   format %{ %}
 5245   interface(REG_INTER);
 5246 %}
 5247 
 5248 // Register R3 only
 5249 operand iRegI_R3()
 5250 %{
 5251   constraint(ALLOC_IN_RC(int_r3_reg));
 5252   match(RegI);
 5253   match(iRegINoSp);
 5254   op_cost(0);
 5255   format %{ %}
 5256   interface(REG_INTER);
 5257 %}
 5258 
 5259 
 5260 // Register R4 only
 5261 operand iRegI_R4()
 5262 %{
 5263   constraint(ALLOC_IN_RC(int_r4_reg));
 5264   match(RegI);
 5265   match(iRegINoSp);
 5266   op_cost(0);
 5267   format %{ %}
 5268   interface(REG_INTER);
 5269 %}
 5270 
 5271 
 5272 // Pointer Register Operands
 5273 // Narrow Pointer Register
 5274 operand iRegN()
 5275 %{
 5276   constraint(ALLOC_IN_RC(any_reg32));
 5277   match(RegN);
 5278   match(iRegNNoSp);
 5279   op_cost(0);
 5280   format %{ %}
 5281   interface(REG_INTER);
 5282 %}
 5283 
 5284 operand iRegN_R0()
 5285 %{
 5286   constraint(ALLOC_IN_RC(r0_reg));
 5287   match(iRegN);
 5288   op_cost(0);
 5289   format %{ %}
 5290   interface(REG_INTER);
 5291 %}
 5292 
 5293 operand iRegN_R2()
 5294 %{
 5295   constraint(ALLOC_IN_RC(r2_reg));
 5296   match(iRegN);
 5297   op_cost(0);
 5298   format %{ %}
 5299   interface(REG_INTER);
 5300 %}
 5301 
 5302 operand iRegN_R3()
 5303 %{
 5304   constraint(ALLOC_IN_RC(r3_reg));
 5305   match(iRegN);
 5306   op_cost(0);
 5307   format %{ %}
 5308   interface(REG_INTER);
 5309 %}
 5310 
 5311 // Integer 64 bit Register not Special
 5312 operand iRegNNoSp()
 5313 %{
 5314   constraint(ALLOC_IN_RC(no_special_reg32));
 5315   match(RegN);
 5316   op_cost(0);
 5317   format %{ %}
 5318   interface(REG_INTER);
 5319 %}
 5320 
 5321 // heap base register -- used for encoding immN0
 5322 
 5323 operand iRegIHeapbase()
 5324 %{
 5325   constraint(ALLOC_IN_RC(heapbase_reg));
 5326   match(RegI);
 5327   op_cost(0);
 5328   format %{ %}
 5329   interface(REG_INTER);
 5330 %}
 5331 
 5332 // Float Register
 5333 // Float register operands
 5334 operand vRegF()
 5335 %{
 5336   constraint(ALLOC_IN_RC(float_reg));
 5337   match(RegF);
 5338 
 5339   op_cost(0);
 5340   format %{ %}
 5341   interface(REG_INTER);
 5342 %}
 5343 
 5344 // Double Register
 5345 // Double register operands
 5346 operand vRegD()
 5347 %{
 5348   constraint(ALLOC_IN_RC(double_reg));
 5349   match(RegD);
 5350 
 5351   op_cost(0);
 5352   format %{ %}
 5353   interface(REG_INTER);
 5354 %}
 5355 
 5356 // Generic vector class. This will be used for
 5357 // all vector operands, including NEON and SVE,
 5358 // but currently only used for SVE VecA.
 5359 operand vReg()
 5360 %{
 5361   constraint(ALLOC_IN_RC(vectora_reg));
 5362   match(VecA);
 5363   op_cost(0);
 5364   format %{ %}
 5365   interface(REG_INTER);
 5366 %}
 5367 
 5368 operand vecD()
 5369 %{
 5370   constraint(ALLOC_IN_RC(vectord_reg));
 5371   match(VecD);
 5372 
 5373   op_cost(0);
 5374   format %{ %}
 5375   interface(REG_INTER);
 5376 %}
 5377 
 5378 operand vecX()
 5379 %{
 5380   constraint(ALLOC_IN_RC(vectorx_reg));
 5381   match(VecX);
 5382 
 5383   op_cost(0);
 5384   format %{ %}
 5385   interface(REG_INTER);
 5386 %}
 5387 
 5388 operand vRegD_V0()
 5389 %{
 5390   constraint(ALLOC_IN_RC(v0_reg));
 5391   match(RegD);
 5392   op_cost(0);
 5393   format %{ %}
 5394   interface(REG_INTER);
 5395 %}
 5396 
 5397 operand vRegD_V1()
 5398 %{
 5399   constraint(ALLOC_IN_RC(v1_reg));
 5400   match(RegD);
 5401   op_cost(0);
 5402   format %{ %}
 5403   interface(REG_INTER);
 5404 %}
 5405 
 5406 operand vRegD_V2()
 5407 %{
 5408   constraint(ALLOC_IN_RC(v2_reg));
 5409   match(RegD);
 5410   op_cost(0);
 5411   format %{ %}
 5412   interface(REG_INTER);
 5413 %}
 5414 
 5415 operand vRegD_V3()
 5416 %{
 5417   constraint(ALLOC_IN_RC(v3_reg));
 5418   match(RegD);
 5419   op_cost(0);
 5420   format %{ %}
 5421   interface(REG_INTER);
 5422 %}
 5423 
 5424 operand vRegD_V4()
 5425 %{
 5426   constraint(ALLOC_IN_RC(v4_reg));
 5427   match(RegD);
 5428   op_cost(0);
 5429   format %{ %}
 5430   interface(REG_INTER);
 5431 %}
 5432 
 5433 operand vRegD_V5()
 5434 %{
 5435   constraint(ALLOC_IN_RC(v5_reg));
 5436   match(RegD);
 5437   op_cost(0);
 5438   format %{ %}
 5439   interface(REG_INTER);
 5440 %}
 5441 
 5442 operand vRegD_V6()
 5443 %{
 5444   constraint(ALLOC_IN_RC(v6_reg));
 5445   match(RegD);
 5446   op_cost(0);
 5447   format %{ %}
 5448   interface(REG_INTER);
 5449 %}
 5450 
 5451 operand vRegD_V7()
 5452 %{
 5453   constraint(ALLOC_IN_RC(v7_reg));
 5454   match(RegD);
 5455   op_cost(0);
 5456   format %{ %}
 5457   interface(REG_INTER);
 5458 %}
 5459 
 5460 operand vRegD_V8()
 5461 %{
 5462   constraint(ALLOC_IN_RC(v8_reg));
 5463   match(RegD);
 5464   op_cost(0);
 5465   format %{ %}
 5466   interface(REG_INTER);
 5467 %}
 5468 
 5469 operand vRegD_V9()
 5470 %{
 5471   constraint(ALLOC_IN_RC(v9_reg));
 5472   match(RegD);
 5473   op_cost(0);
 5474   format %{ %}
 5475   interface(REG_INTER);
 5476 %}
 5477 
 5478 operand vRegD_V10()
 5479 %{
 5480   constraint(ALLOC_IN_RC(v10_reg));
 5481   match(RegD);
 5482   op_cost(0);
 5483   format %{ %}
 5484   interface(REG_INTER);
 5485 %}
 5486 
 5487 operand vRegD_V11()
 5488 %{
 5489   constraint(ALLOC_IN_RC(v11_reg));
 5490   match(RegD);
 5491   op_cost(0);
 5492   format %{ %}
 5493   interface(REG_INTER);
 5494 %}
 5495 
 5496 operand vRegD_V12()
 5497 %{
 5498   constraint(ALLOC_IN_RC(v12_reg));
 5499   match(RegD);
 5500   op_cost(0);
 5501   format %{ %}
 5502   interface(REG_INTER);
 5503 %}
 5504 
 5505 operand vRegD_V13()
 5506 %{
 5507   constraint(ALLOC_IN_RC(v13_reg));
 5508   match(RegD);
 5509   op_cost(0);
 5510   format %{ %}
 5511   interface(REG_INTER);
 5512 %}
 5513 
 5514 operand vRegD_V14()
 5515 %{
 5516   constraint(ALLOC_IN_RC(v14_reg));
 5517   match(RegD);
 5518   op_cost(0);
 5519   format %{ %}
 5520   interface(REG_INTER);
 5521 %}
 5522 
 5523 operand vRegD_V15()
 5524 %{
 5525   constraint(ALLOC_IN_RC(v15_reg));
 5526   match(RegD);
 5527   op_cost(0);
 5528   format %{ %}
 5529   interface(REG_INTER);
 5530 %}
 5531 
 5532 operand vRegD_V16()
 5533 %{
 5534   constraint(ALLOC_IN_RC(v16_reg));
 5535   match(RegD);
 5536   op_cost(0);
 5537   format %{ %}
 5538   interface(REG_INTER);
 5539 %}
 5540 
 5541 operand vRegD_V17()
 5542 %{
 5543   constraint(ALLOC_IN_RC(v17_reg));
 5544   match(RegD);
 5545   op_cost(0);
 5546   format %{ %}
 5547   interface(REG_INTER);
 5548 %}
 5549 
 5550 operand vRegD_V18()
 5551 %{
 5552   constraint(ALLOC_IN_RC(v18_reg));
 5553   match(RegD);
 5554   op_cost(0);
 5555   format %{ %}
 5556   interface(REG_INTER);
 5557 %}
 5558 
 5559 operand vRegD_V19()
 5560 %{
 5561   constraint(ALLOC_IN_RC(v19_reg));
 5562   match(RegD);
 5563   op_cost(0);
 5564   format %{ %}
 5565   interface(REG_INTER);
 5566 %}
 5567 
 5568 operand vRegD_V20()
 5569 %{
 5570   constraint(ALLOC_IN_RC(v20_reg));
 5571   match(RegD);
 5572   op_cost(0);
 5573   format %{ %}
 5574   interface(REG_INTER);
 5575 %}
 5576 
 5577 operand vRegD_V21()
 5578 %{
 5579   constraint(ALLOC_IN_RC(v21_reg));
 5580   match(RegD);
 5581   op_cost(0);
 5582   format %{ %}
 5583   interface(REG_INTER);
 5584 %}
 5585 
 5586 operand vRegD_V22()
 5587 %{
 5588   constraint(ALLOC_IN_RC(v22_reg));
 5589   match(RegD);
 5590   op_cost(0);
 5591   format %{ %}
 5592   interface(REG_INTER);
 5593 %}
 5594 
 5595 operand vRegD_V23()
 5596 %{
 5597   constraint(ALLOC_IN_RC(v23_reg));
 5598   match(RegD);
 5599   op_cost(0);
 5600   format %{ %}
 5601   interface(REG_INTER);
 5602 %}
 5603 
 5604 operand vRegD_V24()
 5605 %{
 5606   constraint(ALLOC_IN_RC(v24_reg));
 5607   match(RegD);
 5608   op_cost(0);
 5609   format %{ %}
 5610   interface(REG_INTER);
 5611 %}
 5612 
 5613 operand vRegD_V25()
 5614 %{
 5615   constraint(ALLOC_IN_RC(v25_reg));
 5616   match(RegD);
 5617   op_cost(0);
 5618   format %{ %}
 5619   interface(REG_INTER);
 5620 %}
 5621 
 5622 operand vRegD_V26()
 5623 %{
 5624   constraint(ALLOC_IN_RC(v26_reg));
 5625   match(RegD);
 5626   op_cost(0);
 5627   format %{ %}
 5628   interface(REG_INTER);
 5629 %}
 5630 
 5631 operand vRegD_V27()
 5632 %{
 5633   constraint(ALLOC_IN_RC(v27_reg));
 5634   match(RegD);
 5635   op_cost(0);
 5636   format %{ %}
 5637   interface(REG_INTER);
 5638 %}
 5639 
 5640 operand vRegD_V28()
 5641 %{
 5642   constraint(ALLOC_IN_RC(v28_reg));
 5643   match(RegD);
 5644   op_cost(0);
 5645   format %{ %}
 5646   interface(REG_INTER);
 5647 %}
 5648 
 5649 operand vRegD_V29()
 5650 %{
 5651   constraint(ALLOC_IN_RC(v29_reg));
 5652   match(RegD);
 5653   op_cost(0);
 5654   format %{ %}
 5655   interface(REG_INTER);
 5656 %}
 5657 
 5658 operand vRegD_V30()
 5659 %{
 5660   constraint(ALLOC_IN_RC(v30_reg));
 5661   match(RegD);
 5662   op_cost(0);
 5663   format %{ %}
 5664   interface(REG_INTER);
 5665 %}
 5666 
 5667 operand vRegD_V31()
 5668 %{
 5669   constraint(ALLOC_IN_RC(v31_reg));
 5670   match(RegD);
 5671   op_cost(0);
 5672   format %{ %}
 5673   interface(REG_INTER);
 5674 %}
 5675 
 5676 operand pReg()
 5677 %{
 5678   constraint(ALLOC_IN_RC(pr_reg));
 5679   match(RegVectMask);
 5680   match(pRegGov);
 5681   op_cost(0);
 5682   format %{ %}
 5683   interface(REG_INTER);
 5684 %}
 5685 
 5686 operand pRegGov()
 5687 %{
 5688   constraint(ALLOC_IN_RC(gov_pr));
 5689   match(RegVectMask);
 5690   op_cost(0);
 5691   format %{ %}
 5692   interface(REG_INTER);
 5693 %}
 5694 
 5695 operand pRegGov_P0()
 5696 %{
 5697   constraint(ALLOC_IN_RC(p0_reg));
 5698   match(RegVectMask);
 5699   op_cost(0);
 5700   format %{ %}
 5701   interface(REG_INTER);
 5702 %}
 5703 
 5704 operand pRegGov_P1()
 5705 %{
 5706   constraint(ALLOC_IN_RC(p1_reg));
 5707   match(RegVectMask);
 5708   op_cost(0);
 5709   format %{ %}
 5710   interface(REG_INTER);
 5711 %}
 5712 
 5713 // Flags register, used as output of signed compare instructions
 5714 
 5715 // note that on AArch64 we also use this register as the output for
 5716 // for floating point compare instructions (CmpF CmpD). this ensures
 5717 // that ordered inequality tests use GT, GE, LT or LE none of which
 5718 // pass through cases where the result is unordered i.e. one or both
 5719 // inputs to the compare is a NaN. this means that the ideal code can
 5720 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5721 // (where the comparison should always fail). EQ and NE tests are
 5722 // always generated in ideal code so that unordered folds into the NE
 5723 // case, matching the behaviour of AArch64 NE.
 5724 //
 5725 // This differs from x86 where the outputs of FP compares use a
 5726 // special FP flags registers and where compares based on this
 5727 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5728 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5729 // to explicitly handle the unordered case in branches. x86 also has
 5730 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5731 
 5732 operand rFlagsReg()
 5733 %{
 5734   constraint(ALLOC_IN_RC(int_flags));
 5735   match(RegFlags);
 5736 
 5737   op_cost(0);
 5738   format %{ "RFLAGS" %}
 5739   interface(REG_INTER);
 5740 %}
 5741 
 5742 // Flags register, used as output of unsigned compare instructions
 5743 operand rFlagsRegU()
 5744 %{
 5745   constraint(ALLOC_IN_RC(int_flags));
 5746   match(RegFlags);
 5747 
 5748   op_cost(0);
 5749   format %{ "RFLAGSU" %}
 5750   interface(REG_INTER);
 5751 %}
 5752 
 5753 // Special Registers
 5754 
 5755 // Method Register
 5756 operand inline_cache_RegP(iRegP reg)
 5757 %{
 5758   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5759   match(reg);
 5760   match(iRegPNoSp);
 5761   op_cost(0);
 5762   format %{ %}
 5763   interface(REG_INTER);
 5764 %}
 5765 
 5766 // Thread Register
 5767 operand thread_RegP(iRegP reg)
 5768 %{
 5769   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5770   match(reg);
 5771   op_cost(0);
 5772   format %{ %}
 5773   interface(REG_INTER);
 5774 %}
 5775 
 5776 operand lr_RegP(iRegP reg)
 5777 %{
 5778   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5779   match(reg);
 5780   op_cost(0);
 5781   format %{ %}
 5782   interface(REG_INTER);
 5783 %}
 5784 
 5785 //----------Memory Operands----------------------------------------------------
 5786 
 5787 operand indirect(iRegP reg)
 5788 %{
 5789   constraint(ALLOC_IN_RC(ptr_reg));
 5790   match(reg);
 5791   op_cost(0);
 5792   format %{ "[$reg]" %}
 5793   interface(MEMORY_INTER) %{
 5794     base($reg);
 5795     index(0xffffffff);
 5796     scale(0x0);
 5797     disp(0x0);
 5798   %}
 5799 %}
 5800 
 5801 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5802 %{
 5803   constraint(ALLOC_IN_RC(ptr_reg));
 5804   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5805   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5806   op_cost(0);
 5807   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5808   interface(MEMORY_INTER) %{
 5809     base($reg);
 5810     index($ireg);
 5811     scale($scale);
 5812     disp(0x0);
 5813   %}
 5814 %}
 5815 
 5816 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5817 %{
 5818   constraint(ALLOC_IN_RC(ptr_reg));
 5819   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5820   match(AddP reg (LShiftL lreg scale));
 5821   op_cost(0);
 5822   format %{ "$reg, $lreg lsl($scale)" %}
 5823   interface(MEMORY_INTER) %{
 5824     base($reg);
 5825     index($lreg);
 5826     scale($scale);
 5827     disp(0x0);
 5828   %}
 5829 %}
 5830 
 5831 operand indIndexI2L(iRegP reg, iRegI ireg)
 5832 %{
 5833   constraint(ALLOC_IN_RC(ptr_reg));
 5834   match(AddP reg (ConvI2L ireg));
 5835   op_cost(0);
 5836   format %{ "$reg, $ireg, 0, I2L" %}
 5837   interface(MEMORY_INTER) %{
 5838     base($reg);
 5839     index($ireg);
 5840     scale(0x0);
 5841     disp(0x0);
 5842   %}
 5843 %}
 5844 
 5845 operand indIndex(iRegP reg, iRegL lreg)
 5846 %{
 5847   constraint(ALLOC_IN_RC(ptr_reg));
 5848   match(AddP reg lreg);
 5849   op_cost(0);
 5850   format %{ "$reg, $lreg" %}
 5851   interface(MEMORY_INTER) %{
 5852     base($reg);
 5853     index($lreg);
 5854     scale(0x0);
 5855     disp(0x0);
 5856   %}
 5857 %}
 5858 
 5859 operand indOffI(iRegP reg, immIOffset off)
 5860 %{
 5861   constraint(ALLOC_IN_RC(ptr_reg));
 5862   match(AddP reg off);
 5863   op_cost(0);
 5864   format %{ "[$reg, $off]" %}
 5865   interface(MEMORY_INTER) %{
 5866     base($reg);
 5867     index(0xffffffff);
 5868     scale(0x0);
 5869     disp($off);
 5870   %}
 5871 %}
 5872 
 5873 operand indOffI1(iRegP reg, immIOffset1 off)
 5874 %{
 5875   constraint(ALLOC_IN_RC(ptr_reg));
 5876   match(AddP reg off);
 5877   op_cost(0);
 5878   format %{ "[$reg, $off]" %}
 5879   interface(MEMORY_INTER) %{
 5880     base($reg);
 5881     index(0xffffffff);
 5882     scale(0x0);
 5883     disp($off);
 5884   %}
 5885 %}
 5886 
 5887 operand indOffI2(iRegP reg, immIOffset2 off)
 5888 %{
 5889   constraint(ALLOC_IN_RC(ptr_reg));
 5890   match(AddP reg off);
 5891   op_cost(0);
 5892   format %{ "[$reg, $off]" %}
 5893   interface(MEMORY_INTER) %{
 5894     base($reg);
 5895     index(0xffffffff);
 5896     scale(0x0);
 5897     disp($off);
 5898   %}
 5899 %}
 5900 
 5901 operand indOffI4(iRegP reg, immIOffset4 off)
 5902 %{
 5903   constraint(ALLOC_IN_RC(ptr_reg));
 5904   match(AddP reg off);
 5905   op_cost(0);
 5906   format %{ "[$reg, $off]" %}
 5907   interface(MEMORY_INTER) %{
 5908     base($reg);
 5909     index(0xffffffff);
 5910     scale(0x0);
 5911     disp($off);
 5912   %}
 5913 %}
 5914 
 5915 operand indOffI8(iRegP reg, immIOffset8 off)
 5916 %{
 5917   constraint(ALLOC_IN_RC(ptr_reg));
 5918   match(AddP reg off);
 5919   op_cost(0);
 5920   format %{ "[$reg, $off]" %}
 5921   interface(MEMORY_INTER) %{
 5922     base($reg);
 5923     index(0xffffffff);
 5924     scale(0x0);
 5925     disp($off);
 5926   %}
 5927 %}
 5928 
 5929 operand indOffI16(iRegP reg, immIOffset16 off)
 5930 %{
 5931   constraint(ALLOC_IN_RC(ptr_reg));
 5932   match(AddP reg off);
 5933   op_cost(0);
 5934   format %{ "[$reg, $off]" %}
 5935   interface(MEMORY_INTER) %{
 5936     base($reg);
 5937     index(0xffffffff);
 5938     scale(0x0);
 5939     disp($off);
 5940   %}
 5941 %}
 5942 
 5943 operand indOffL(iRegP reg, immLoffset off)
 5944 %{
 5945   constraint(ALLOC_IN_RC(ptr_reg));
 5946   match(AddP reg off);
 5947   op_cost(0);
 5948   format %{ "[$reg, $off]" %}
 5949   interface(MEMORY_INTER) %{
 5950     base($reg);
 5951     index(0xffffffff);
 5952     scale(0x0);
 5953     disp($off);
 5954   %}
 5955 %}
 5956 
 5957 operand indOffL1(iRegP reg, immLoffset1 off)
 5958 %{
 5959   constraint(ALLOC_IN_RC(ptr_reg));
 5960   match(AddP reg off);
 5961   op_cost(0);
 5962   format %{ "[$reg, $off]" %}
 5963   interface(MEMORY_INTER) %{
 5964     base($reg);
 5965     index(0xffffffff);
 5966     scale(0x0);
 5967     disp($off);
 5968   %}
 5969 %}
 5970 
 5971 operand indOffL2(iRegP reg, immLoffset2 off)
 5972 %{
 5973   constraint(ALLOC_IN_RC(ptr_reg));
 5974   match(AddP reg off);
 5975   op_cost(0);
 5976   format %{ "[$reg, $off]" %}
 5977   interface(MEMORY_INTER) %{
 5978     base($reg);
 5979     index(0xffffffff);
 5980     scale(0x0);
 5981     disp($off);
 5982   %}
 5983 %}
 5984 
 5985 operand indOffL4(iRegP reg, immLoffset4 off)
 5986 %{
 5987   constraint(ALLOC_IN_RC(ptr_reg));
 5988   match(AddP reg off);
 5989   op_cost(0);
 5990   format %{ "[$reg, $off]" %}
 5991   interface(MEMORY_INTER) %{
 5992     base($reg);
 5993     index(0xffffffff);
 5994     scale(0x0);
 5995     disp($off);
 5996   %}
 5997 %}
 5998 
 5999 operand indOffL8(iRegP reg, immLoffset8 off)
 6000 %{
 6001   constraint(ALLOC_IN_RC(ptr_reg));
 6002   match(AddP reg off);
 6003   op_cost(0);
 6004   format %{ "[$reg, $off]" %}
 6005   interface(MEMORY_INTER) %{
 6006     base($reg);
 6007     index(0xffffffff);
 6008     scale(0x0);
 6009     disp($off);
 6010   %}
 6011 %}
 6012 
 6013 operand indOffL16(iRegP reg, immLoffset16 off)
 6014 %{
 6015   constraint(ALLOC_IN_RC(ptr_reg));
 6016   match(AddP reg off);
 6017   op_cost(0);
 6018   format %{ "[$reg, $off]" %}
 6019   interface(MEMORY_INTER) %{
 6020     base($reg);
 6021     index(0xffffffff);
 6022     scale(0x0);
 6023     disp($off);
 6024   %}
 6025 %}
 6026 
 6027 operand indirectN(iRegN reg)
 6028 %{
 6029   predicate(CompressedOops::shift() == 0);
 6030   constraint(ALLOC_IN_RC(ptr_reg));
 6031   match(DecodeN reg);
 6032   op_cost(0);
 6033   format %{ "[$reg]\t# narrow" %}
 6034   interface(MEMORY_INTER) %{
 6035     base($reg);
 6036     index(0xffffffff);
 6037     scale(0x0);
 6038     disp(0x0);
 6039   %}
 6040 %}
 6041 
 6042 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6043 %{
 6044   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6045   constraint(ALLOC_IN_RC(ptr_reg));
 6046   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6047   op_cost(0);
 6048   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6049   interface(MEMORY_INTER) %{
 6050     base($reg);
 6051     index($ireg);
 6052     scale($scale);
 6053     disp(0x0);
 6054   %}
 6055 %}
 6056 
 6057 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6058 %{
 6059   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6060   constraint(ALLOC_IN_RC(ptr_reg));
 6061   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6062   op_cost(0);
 6063   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6064   interface(MEMORY_INTER) %{
 6065     base($reg);
 6066     index($lreg);
 6067     scale($scale);
 6068     disp(0x0);
 6069   %}
 6070 %}
 6071 
 6072 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6073 %{
 6074   predicate(CompressedOops::shift() == 0);
 6075   constraint(ALLOC_IN_RC(ptr_reg));
 6076   match(AddP (DecodeN reg) (ConvI2L ireg));
 6077   op_cost(0);
 6078   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6079   interface(MEMORY_INTER) %{
 6080     base($reg);
 6081     index($ireg);
 6082     scale(0x0);
 6083     disp(0x0);
 6084   %}
 6085 %}
 6086 
 6087 operand indIndexN(iRegN reg, iRegL lreg)
 6088 %{
 6089   predicate(CompressedOops::shift() == 0);
 6090   constraint(ALLOC_IN_RC(ptr_reg));
 6091   match(AddP (DecodeN reg) lreg);
 6092   op_cost(0);
 6093   format %{ "$reg, $lreg\t# narrow" %}
 6094   interface(MEMORY_INTER) %{
 6095     base($reg);
 6096     index($lreg);
 6097     scale(0x0);
 6098     disp(0x0);
 6099   %}
 6100 %}
 6101 
 6102 operand indOffIN(iRegN reg, immIOffset off)
 6103 %{
 6104   predicate(CompressedOops::shift() == 0);
 6105   constraint(ALLOC_IN_RC(ptr_reg));
 6106   match(AddP (DecodeN reg) off);
 6107   op_cost(0);
 6108   format %{ "[$reg, $off]\t# narrow" %}
 6109   interface(MEMORY_INTER) %{
 6110     base($reg);
 6111     index(0xffffffff);
 6112     scale(0x0);
 6113     disp($off);
 6114   %}
 6115 %}
 6116 
 6117 operand indOffLN(iRegN reg, immLoffset off)
 6118 %{
 6119   predicate(CompressedOops::shift() == 0);
 6120   constraint(ALLOC_IN_RC(ptr_reg));
 6121   match(AddP (DecodeN reg) off);
 6122   op_cost(0);
 6123   format %{ "[$reg, $off]\t# narrow" %}
 6124   interface(MEMORY_INTER) %{
 6125     base($reg);
 6126     index(0xffffffff);
 6127     scale(0x0);
 6128     disp($off);
 6129   %}
 6130 %}
 6131 
 6132 
 6133 
 6134 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6135 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6136 %{
 6137   constraint(ALLOC_IN_RC(ptr_reg));
 6138   match(AddP reg off);
 6139   op_cost(0);
 6140   format %{ "[$reg, $off]" %}
 6141   interface(MEMORY_INTER) %{
 6142     base($reg);
 6143     index(0xffffffff);
 6144     scale(0x0);
 6145     disp($off);
 6146   %}
 6147 %}
 6148 
 6149 //----------Special Memory Operands--------------------------------------------
 6150 // Stack Slot Operand - This operand is used for loading and storing temporary
 6151 //                      values on the stack where a match requires a value to
 6152 //                      flow through memory.
 6153 operand stackSlotP(sRegP reg)
 6154 %{
 6155   constraint(ALLOC_IN_RC(stack_slots));
 6156   op_cost(100);
 6157   // No match rule because this operand is only generated in matching
 6158   // match(RegP);
 6159   format %{ "[$reg]" %}
 6160   interface(MEMORY_INTER) %{
 6161     base(0x1e);  // RSP
 6162     index(0x0);  // No Index
 6163     scale(0x0);  // No Scale
 6164     disp($reg);  // Stack Offset
 6165   %}
 6166 %}
 6167 
 6168 operand stackSlotI(sRegI reg)
 6169 %{
 6170   constraint(ALLOC_IN_RC(stack_slots));
 6171   // No match rule because this operand is only generated in matching
 6172   // match(RegI);
 6173   format %{ "[$reg]" %}
 6174   interface(MEMORY_INTER) %{
 6175     base(0x1e);  // RSP
 6176     index(0x0);  // No Index
 6177     scale(0x0);  // No Scale
 6178     disp($reg);  // Stack Offset
 6179   %}
 6180 %}
 6181 
 6182 operand stackSlotF(sRegF reg)
 6183 %{
 6184   constraint(ALLOC_IN_RC(stack_slots));
 6185   // No match rule because this operand is only generated in matching
 6186   // match(RegF);
 6187   format %{ "[$reg]" %}
 6188   interface(MEMORY_INTER) %{
 6189     base(0x1e);  // RSP
 6190     index(0x0);  // No Index
 6191     scale(0x0);  // No Scale
 6192     disp($reg);  // Stack Offset
 6193   %}
 6194 %}
 6195 
 6196 operand stackSlotD(sRegD reg)
 6197 %{
 6198   constraint(ALLOC_IN_RC(stack_slots));
 6199   // No match rule because this operand is only generated in matching
 6200   // match(RegD);
 6201   format %{ "[$reg]" %}
 6202   interface(MEMORY_INTER) %{
 6203     base(0x1e);  // RSP
 6204     index(0x0);  // No Index
 6205     scale(0x0);  // No Scale
 6206     disp($reg);  // Stack Offset
 6207   %}
 6208 %}
 6209 
 6210 operand stackSlotL(sRegL reg)
 6211 %{
 6212   constraint(ALLOC_IN_RC(stack_slots));
 6213   // No match rule because this operand is only generated in matching
 6214   // match(RegL);
 6215   format %{ "[$reg]" %}
 6216   interface(MEMORY_INTER) %{
 6217     base(0x1e);  // RSP
 6218     index(0x0);  // No Index
 6219     scale(0x0);  // No Scale
 6220     disp($reg);  // Stack Offset
 6221   %}
 6222 %}
 6223 
 6224 // Operands for expressing Control Flow
 6225 // NOTE: Label is a predefined operand which should not be redefined in
 6226 //       the AD file. It is generically handled within the ADLC.
 6227 
 6228 //----------Conditional Branch Operands----------------------------------------
 6229 // Comparison Op  - This is the operation of the comparison, and is limited to
 6230 //                  the following set of codes:
 6231 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6232 //
 6233 // Other attributes of the comparison, such as unsignedness, are specified
 6234 // by the comparison instruction that sets a condition code flags register.
 6235 // That result is represented by a flags operand whose subtype is appropriate
 6236 // to the unsignedness (etc.) of the comparison.
 6237 //
 6238 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6239 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6240 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6241 
 6242 // used for signed integral comparisons and fp comparisons
 6243 
 6244 operand cmpOp()
 6245 %{
 6246   match(Bool);
 6247 
 6248   format %{ "" %}
 6249   interface(COND_INTER) %{
 6250     equal(0x0, "eq");
 6251     not_equal(0x1, "ne");
 6252     less(0xb, "lt");
 6253     greater_equal(0xa, "ge");
 6254     less_equal(0xd, "le");
 6255     greater(0xc, "gt");
 6256     overflow(0x6, "vs");
 6257     no_overflow(0x7, "vc");
 6258   %}
 6259 %}
 6260 
 6261 // used for unsigned integral comparisons
 6262 
 6263 operand cmpOpU()
 6264 %{
 6265   match(Bool);
 6266 
 6267   format %{ "" %}
 6268   interface(COND_INTER) %{
 6269     equal(0x0, "eq");
 6270     not_equal(0x1, "ne");
 6271     less(0x3, "lo");
 6272     greater_equal(0x2, "hs");
 6273     less_equal(0x9, "ls");
 6274     greater(0x8, "hi");
 6275     overflow(0x6, "vs");
 6276     no_overflow(0x7, "vc");
 6277   %}
 6278 %}
 6279 
 6280 // used for certain integral comparisons which can be
 6281 // converted to cbxx or tbxx instructions
 6282 
 6283 operand cmpOpEqNe()
 6284 %{
 6285   match(Bool);
 6286   op_cost(0);
 6287   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6288             || n->as_Bool()->_test._test == BoolTest::eq);
 6289 
 6290   format %{ "" %}
 6291   interface(COND_INTER) %{
 6292     equal(0x0, "eq");
 6293     not_equal(0x1, "ne");
 6294     less(0xb, "lt");
 6295     greater_equal(0xa, "ge");
 6296     less_equal(0xd, "le");
 6297     greater(0xc, "gt");
 6298     overflow(0x6, "vs");
 6299     no_overflow(0x7, "vc");
 6300   %}
 6301 %}
 6302 
 6303 // used for certain integral comparisons which can be
 6304 // converted to cbxx or tbxx instructions
 6305 
 6306 operand cmpOpLtGe()
 6307 %{
 6308   match(Bool);
 6309   op_cost(0);
 6310 
 6311   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6312             || n->as_Bool()->_test._test == BoolTest::ge);
 6313 
 6314   format %{ "" %}
 6315   interface(COND_INTER) %{
 6316     equal(0x0, "eq");
 6317     not_equal(0x1, "ne");
 6318     less(0xb, "lt");
 6319     greater_equal(0xa, "ge");
 6320     less_equal(0xd, "le");
 6321     greater(0xc, "gt");
 6322     overflow(0x6, "vs");
 6323     no_overflow(0x7, "vc");
 6324   %}
 6325 %}
 6326 
 6327 // used for certain unsigned integral comparisons which can be
 6328 // converted to cbxx or tbxx instructions
 6329 
 6330 operand cmpOpUEqNeLtGe()
 6331 %{
 6332   match(Bool);
 6333   op_cost(0);
 6334 
 6335   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6336             || n->as_Bool()->_test._test == BoolTest::ne
 6337             || n->as_Bool()->_test._test == BoolTest::lt
 6338             || n->as_Bool()->_test._test == BoolTest::ge);
 6339 
 6340   format %{ "" %}
 6341   interface(COND_INTER) %{
 6342     equal(0x0, "eq");
 6343     not_equal(0x1, "ne");
 6344     less(0xb, "lt");
 6345     greater_equal(0xa, "ge");
 6346     less_equal(0xd, "le");
 6347     greater(0xc, "gt");
 6348     overflow(0x6, "vs");
 6349     no_overflow(0x7, "vc");
 6350   %}
 6351 %}
 6352 
 6353 // Special operand allowing long args to int ops to be truncated for free
 6354 
 6355 operand iRegL2I(iRegL reg) %{
 6356 
 6357   op_cost(0);
 6358 
 6359   match(ConvL2I reg);
 6360 
 6361   format %{ "l2i($reg)" %}
 6362 
 6363   interface(REG_INTER)
 6364 %}
 6365 
 6366 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6367 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6368 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6369 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6370 
 6371 //----------OPERAND CLASSES----------------------------------------------------
 6372 // Operand Classes are groups of operands that are used as to simplify
 6373 // instruction definitions by not requiring the AD writer to specify
 6374 // separate instructions for every form of operand when the
 6375 // instruction accepts multiple operand types with the same basic
 6376 // encoding and format. The classic case of this is memory operands.
 6377 
 6378 // memory is used to define read/write location for load/store
 6379 // instruction defs. we can turn a memory op into an Address
 6380 
 6381 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6382                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6383 
 6384 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6385                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6386 
 6387 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6388                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6389 
 6390 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6391                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6392 
 6393 // All of the memory operands. For the pipeline description.
 6394 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6395                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6396                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6397 
 6398 
 6399 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6400 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6401 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6402 // can be elided because the 32-bit instruction will just employ the
 6403 // lower 32 bits anyway.
 6404 //
 6405 // n.b. this does not elide all L2I conversions. if the truncated
 6406 // value is consumed by more than one operation then the ConvL2I
 6407 // cannot be bundled into the consuming nodes so an l2i gets planted
 6408 // (actually a movw $dst $src) and the downstream instructions consume
 6409 // the result of the l2i as an iRegI input. That's a shame since the
 6410 // movw is actually redundant but its not too costly.
 6411 
 6412 opclass iRegIorL2I(iRegI, iRegL2I);
 6413 
 6414 //----------PIPELINE-----------------------------------------------------------
 6415 // Rules which define the behavior of the target architectures pipeline.
 6416 
 6417 // For specific pipelines, eg A53, define the stages of that pipeline
 6418 //pipe_desc(ISS, EX1, EX2, WR);
 6419 #define ISS S0
 6420 #define EX1 S1
 6421 #define EX2 S2
 6422 #define WR  S3
 6423 
 6424 // Integer ALU reg operation
 6425 pipeline %{
 6426 
 6427 attributes %{
 6428   // ARM instructions are of fixed length
 6429   fixed_size_instructions;        // Fixed size instructions TODO does
 6430   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6431   // ARM instructions come in 32-bit word units
 6432   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6433   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6434   instruction_fetch_units = 1;       // of 64 bytes
 6435 
 6436   // List of nop instructions
 6437   nops( MachNop );
 6438 %}
 6439 
 6440 // We don't use an actual pipeline model so don't care about resources
 6441 // or description. we do use pipeline classes to introduce fixed
 6442 // latencies
 6443 
 6444 //----------RESOURCES----------------------------------------------------------
 6445 // Resources are the functional units available to the machine
 6446 
 6447 resources( INS0, INS1, INS01 = INS0 | INS1,
 6448            ALU0, ALU1, ALU = ALU0 | ALU1,
 6449            MAC,
 6450            DIV,
 6451            BRANCH,
 6452            LDST,
 6453            NEON_FP);
 6454 
 6455 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6456 // Pipeline Description specifies the stages in the machine's pipeline
 6457 
 6458 // Define the pipeline as a generic 6 stage pipeline
 6459 pipe_desc(S0, S1, S2, S3, S4, S5);
 6460 
 6461 //----------PIPELINE CLASSES---------------------------------------------------
 6462 // Pipeline Classes describe the stages in which input and output are
 6463 // referenced by the hardware pipeline.
 6464 
 6465 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6466 %{
 6467   single_instruction;
 6468   src1   : S1(read);
 6469   src2   : S2(read);
 6470   dst    : S5(write);
 6471   INS01  : ISS;
 6472   NEON_FP : S5;
 6473 %}
 6474 
 6475 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6476 %{
 6477   single_instruction;
 6478   src1   : S1(read);
 6479   src2   : S2(read);
 6480   dst    : S5(write);
 6481   INS01  : ISS;
 6482   NEON_FP : S5;
 6483 %}
 6484 
 6485 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6486 %{
 6487   single_instruction;
 6488   src    : S1(read);
 6489   dst    : S5(write);
 6490   INS01  : ISS;
 6491   NEON_FP : S5;
 6492 %}
 6493 
 6494 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6495 %{
 6496   single_instruction;
 6497   src    : S1(read);
 6498   dst    : S5(write);
 6499   INS01  : ISS;
 6500   NEON_FP : S5;
 6501 %}
 6502 
 6503 pipe_class fp_d2f(vRegF dst, vRegD src)
 6504 %{
 6505   single_instruction;
 6506   src    : S1(read);
 6507   dst    : S5(write);
 6508   INS01  : ISS;
 6509   NEON_FP : S5;
 6510 %}
 6511 
 6512 pipe_class fp_f2d(vRegD dst, vRegF src)
 6513 %{
 6514   single_instruction;
 6515   src    : S1(read);
 6516   dst    : S5(write);
 6517   INS01  : ISS;
 6518   NEON_FP : S5;
 6519 %}
 6520 
 6521 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6522 %{
 6523   single_instruction;
 6524   src    : S1(read);
 6525   dst    : S5(write);
 6526   INS01  : ISS;
 6527   NEON_FP : S5;
 6528 %}
 6529 
 6530 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6531 %{
 6532   single_instruction;
 6533   src    : S1(read);
 6534   dst    : S5(write);
 6535   INS01  : ISS;
 6536   NEON_FP : S5;
 6537 %}
 6538 
 6539 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6540 %{
 6541   single_instruction;
 6542   src    : S1(read);
 6543   dst    : S5(write);
 6544   INS01  : ISS;
 6545   NEON_FP : S5;
 6546 %}
 6547 
 6548 pipe_class fp_l2f(vRegF dst, iRegL src)
 6549 %{
 6550   single_instruction;
 6551   src    : S1(read);
 6552   dst    : S5(write);
 6553   INS01  : ISS;
 6554   NEON_FP : S5;
 6555 %}
 6556 
 6557 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6558 %{
 6559   single_instruction;
 6560   src    : S1(read);
 6561   dst    : S5(write);
 6562   INS01  : ISS;
 6563   NEON_FP : S5;
 6564 %}
 6565 
 6566 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6567 %{
 6568   single_instruction;
 6569   src    : S1(read);
 6570   dst    : S5(write);
 6571   INS01  : ISS;
 6572   NEON_FP : S5;
 6573 %}
 6574 
 6575 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6576 %{
 6577   single_instruction;
 6578   src    : S1(read);
 6579   dst    : S5(write);
 6580   INS01  : ISS;
 6581   NEON_FP : S5;
 6582 %}
 6583 
 6584 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6585 %{
 6586   single_instruction;
 6587   src    : S1(read);
 6588   dst    : S5(write);
 6589   INS01  : ISS;
 6590   NEON_FP : S5;
 6591 %}
 6592 
 6593 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6594 %{
 6595   single_instruction;
 6596   src1   : S1(read);
 6597   src2   : S2(read);
 6598   dst    : S5(write);
 6599   INS0   : ISS;
 6600   NEON_FP : S5;
 6601 %}
 6602 
 6603 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6604 %{
 6605   single_instruction;
 6606   src1   : S1(read);
 6607   src2   : S2(read);
 6608   dst    : S5(write);
 6609   INS0   : ISS;
 6610   NEON_FP : S5;
 6611 %}
 6612 
 6613 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6614 %{
 6615   single_instruction;
 6616   cr     : S1(read);
 6617   src1   : S1(read);
 6618   src2   : S1(read);
 6619   dst    : S3(write);
 6620   INS01  : ISS;
 6621   NEON_FP : S3;
 6622 %}
 6623 
 6624 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6625 %{
 6626   single_instruction;
 6627   cr     : S1(read);
 6628   src1   : S1(read);
 6629   src2   : S1(read);
 6630   dst    : S3(write);
 6631   INS01  : ISS;
 6632   NEON_FP : S3;
 6633 %}
 6634 
 6635 pipe_class fp_imm_s(vRegF dst)
 6636 %{
 6637   single_instruction;
 6638   dst    : S3(write);
 6639   INS01  : ISS;
 6640   NEON_FP : S3;
 6641 %}
 6642 
 6643 pipe_class fp_imm_d(vRegD dst)
 6644 %{
 6645   single_instruction;
 6646   dst    : S3(write);
 6647   INS01  : ISS;
 6648   NEON_FP : S3;
 6649 %}
 6650 
 6651 pipe_class fp_load_constant_s(vRegF dst)
 6652 %{
 6653   single_instruction;
 6654   dst    : S4(write);
 6655   INS01  : ISS;
 6656   NEON_FP : S4;
 6657 %}
 6658 
 6659 pipe_class fp_load_constant_d(vRegD dst)
 6660 %{
 6661   single_instruction;
 6662   dst    : S4(write);
 6663   INS01  : ISS;
 6664   NEON_FP : S4;
 6665 %}
 6666 
 6667 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6668 %{
 6669   single_instruction;
 6670   dst    : S5(write);
 6671   src1   : S1(read);
 6672   src2   : S1(read);
 6673   INS01  : ISS;
 6674   NEON_FP : S5;
 6675 %}
 6676 
 6677 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6678 %{
 6679   single_instruction;
 6680   dst    : S5(write);
 6681   src1   : S1(read);
 6682   src2   : S1(read);
 6683   INS0   : ISS;
 6684   NEON_FP : S5;
 6685 %}
 6686 
 6687 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6688 %{
 6689   single_instruction;
 6690   dst    : S5(write);
 6691   src1   : S1(read);
 6692   src2   : S1(read);
 6693   dst    : S1(read);
 6694   INS01  : ISS;
 6695   NEON_FP : S5;
 6696 %}
 6697 
 6698 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6699 %{
 6700   single_instruction;
 6701   dst    : S5(write);
 6702   src1   : S1(read);
 6703   src2   : S1(read);
 6704   dst    : S1(read);
 6705   INS0   : ISS;
 6706   NEON_FP : S5;
 6707 %}
 6708 
 6709 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6710 %{
 6711   single_instruction;
 6712   dst    : S4(write);
 6713   src1   : S2(read);
 6714   src2   : S2(read);
 6715   INS01  : ISS;
 6716   NEON_FP : S4;
 6717 %}
 6718 
 6719 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6720 %{
 6721   single_instruction;
 6722   dst    : S4(write);
 6723   src1   : S2(read);
 6724   src2   : S2(read);
 6725   INS0   : ISS;
 6726   NEON_FP : S4;
 6727 %}
 6728 
 6729 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6730 %{
 6731   single_instruction;
 6732   dst    : S3(write);
 6733   src1   : S2(read);
 6734   src2   : S2(read);
 6735   INS01  : ISS;
 6736   NEON_FP : S3;
 6737 %}
 6738 
 6739 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6740 %{
 6741   single_instruction;
 6742   dst    : S3(write);
 6743   src1   : S2(read);
 6744   src2   : S2(read);
 6745   INS0   : ISS;
 6746   NEON_FP : S3;
 6747 %}
 6748 
 6749 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6750 %{
 6751   single_instruction;
 6752   dst    : S3(write);
 6753   src    : S1(read);
 6754   shift  : S1(read);
 6755   INS01  : ISS;
 6756   NEON_FP : S3;
 6757 %}
 6758 
 6759 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6760 %{
 6761   single_instruction;
 6762   dst    : S3(write);
 6763   src    : S1(read);
 6764   shift  : S1(read);
 6765   INS0   : ISS;
 6766   NEON_FP : S3;
 6767 %}
 6768 
 6769 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6770 %{
 6771   single_instruction;
 6772   dst    : S3(write);
 6773   src    : S1(read);
 6774   INS01  : ISS;
 6775   NEON_FP : S3;
 6776 %}
 6777 
 6778 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6779 %{
 6780   single_instruction;
 6781   dst    : S3(write);
 6782   src    : S1(read);
 6783   INS0   : ISS;
 6784   NEON_FP : S3;
 6785 %}
 6786 
 6787 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6788 %{
 6789   single_instruction;
 6790   dst    : S5(write);
 6791   src1   : S1(read);
 6792   src2   : S1(read);
 6793   INS01  : ISS;
 6794   NEON_FP : S5;
 6795 %}
 6796 
 6797 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6798 %{
 6799   single_instruction;
 6800   dst    : S5(write);
 6801   src1   : S1(read);
 6802   src2   : S1(read);
 6803   INS0   : ISS;
 6804   NEON_FP : S5;
 6805 %}
 6806 
 6807 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6808 %{
 6809   single_instruction;
 6810   dst    : S5(write);
 6811   src1   : S1(read);
 6812   src2   : S1(read);
 6813   INS0   : ISS;
 6814   NEON_FP : S5;
 6815 %}
 6816 
 6817 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6818 %{
 6819   single_instruction;
 6820   dst    : S5(write);
 6821   src1   : S1(read);
 6822   src2   : S1(read);
 6823   INS0   : ISS;
 6824   NEON_FP : S5;
 6825 %}
 6826 
 6827 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6828 %{
 6829   single_instruction;
 6830   dst    : S5(write);
 6831   src    : S1(read);
 6832   INS0   : ISS;
 6833   NEON_FP : S5;
 6834 %}
 6835 
 6836 pipe_class vunop_fp64(vecD dst, vecD src)
 6837 %{
 6838   single_instruction;
 6839   dst    : S5(write);
 6840   src    : S1(read);
 6841   INS01  : ISS;
 6842   NEON_FP : S5;
 6843 %}
 6844 
 6845 pipe_class vunop_fp128(vecX dst, vecX src)
 6846 %{
 6847   single_instruction;
 6848   dst    : S5(write);
 6849   src    : S1(read);
 6850   INS0   : ISS;
 6851   NEON_FP : S5;
 6852 %}
 6853 
 6854 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6855 %{
 6856   single_instruction;
 6857   dst    : S3(write);
 6858   src    : S1(read);
 6859   INS01  : ISS;
 6860   NEON_FP : S3;
 6861 %}
 6862 
 6863 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6864 %{
 6865   single_instruction;
 6866   dst    : S3(write);
 6867   src    : S1(read);
 6868   INS01  : ISS;
 6869   NEON_FP : S3;
 6870 %}
 6871 
 6872 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6873 %{
 6874   single_instruction;
 6875   dst    : S3(write);
 6876   src    : S1(read);
 6877   INS01  : ISS;
 6878   NEON_FP : S3;
 6879 %}
 6880 
 6881 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6882 %{
 6883   single_instruction;
 6884   dst    : S3(write);
 6885   src    : S1(read);
 6886   INS01  : ISS;
 6887   NEON_FP : S3;
 6888 %}
 6889 
 6890 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6891 %{
 6892   single_instruction;
 6893   dst    : S3(write);
 6894   src    : S1(read);
 6895   INS01  : ISS;
 6896   NEON_FP : S3;
 6897 %}
 6898 
 6899 pipe_class vmovi_reg_imm64(vecD dst)
 6900 %{
 6901   single_instruction;
 6902   dst    : S3(write);
 6903   INS01  : ISS;
 6904   NEON_FP : S3;
 6905 %}
 6906 
 6907 pipe_class vmovi_reg_imm128(vecX dst)
 6908 %{
 6909   single_instruction;
 6910   dst    : S3(write);
 6911   INS0   : ISS;
 6912   NEON_FP : S3;
 6913 %}
 6914 
 6915 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6916 %{
 6917   single_instruction;
 6918   dst    : S5(write);
 6919   mem    : ISS(read);
 6920   INS01  : ISS;
 6921   NEON_FP : S3;
 6922 %}
 6923 
 6924 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6925 %{
 6926   single_instruction;
 6927   dst    : S5(write);
 6928   mem    : ISS(read);
 6929   INS01  : ISS;
 6930   NEON_FP : S3;
 6931 %}
 6932 
 6933 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6934 %{
 6935   single_instruction;
 6936   mem    : ISS(read);
 6937   src    : S2(read);
 6938   INS01  : ISS;
 6939   NEON_FP : S3;
 6940 %}
 6941 
 6942 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6943 %{
 6944   single_instruction;
 6945   mem    : ISS(read);
 6946   src    : S2(read);
 6947   INS01  : ISS;
 6948   NEON_FP : S3;
 6949 %}
 6950 
 6951 //------- Integer ALU operations --------------------------
 6952 
 6953 // Integer ALU reg-reg operation
 6954 // Operands needed in EX1, result generated in EX2
 6955 // Eg.  ADD     x0, x1, x2
 6956 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6957 %{
 6958   single_instruction;
 6959   dst    : EX2(write);
 6960   src1   : EX1(read);
 6961   src2   : EX1(read);
 6962   INS01  : ISS; // Dual issue as instruction 0 or 1
 6963   ALU    : EX2;
 6964 %}
 6965 
 6966 // Integer ALU reg-reg operation with constant shift
 6967 // Shifted register must be available in LATE_ISS instead of EX1
 6968 // Eg.  ADD     x0, x1, x2, LSL #2
 6969 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6970 %{
 6971   single_instruction;
 6972   dst    : EX2(write);
 6973   src1   : EX1(read);
 6974   src2   : ISS(read);
 6975   INS01  : ISS;
 6976   ALU    : EX2;
 6977 %}
 6978 
 6979 // Integer ALU reg operation with constant shift
 6980 // Eg.  LSL     x0, x1, #shift
 6981 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6982 %{
 6983   single_instruction;
 6984   dst    : EX2(write);
 6985   src1   : ISS(read);
 6986   INS01  : ISS;
 6987   ALU    : EX2;
 6988 %}
 6989 
 6990 // Integer ALU reg-reg operation with variable shift
 6991 // Both operands must be available in LATE_ISS instead of EX1
 6992 // Result is available in EX1 instead of EX2
 6993 // Eg.  LSLV    x0, x1, x2
 6994 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6995 %{
 6996   single_instruction;
 6997   dst    : EX1(write);
 6998   src1   : ISS(read);
 6999   src2   : ISS(read);
 7000   INS01  : ISS;
 7001   ALU    : EX1;
 7002 %}
 7003 
 7004 // Integer ALU reg-reg operation with extract
 7005 // As for _vshift above, but result generated in EX2
 7006 // Eg.  EXTR    x0, x1, x2, #N
 7007 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 7008 %{
 7009   single_instruction;
 7010   dst    : EX2(write);
 7011   src1   : ISS(read);
 7012   src2   : ISS(read);
 7013   INS1   : ISS; // Can only dual issue as Instruction 1
 7014   ALU    : EX1;
 7015 %}
 7016 
 7017 // Integer ALU reg operation
 7018 // Eg.  NEG     x0, x1
 7019 pipe_class ialu_reg(iRegI dst, iRegI src)
 7020 %{
 7021   single_instruction;
 7022   dst    : EX2(write);
 7023   src    : EX1(read);
 7024   INS01  : ISS;
 7025   ALU    : EX2;
 7026 %}
 7027 
 7028 // Integer ALU reg mmediate operation
 7029 // Eg.  ADD     x0, x1, #N
 7030 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 7031 %{
 7032   single_instruction;
 7033   dst    : EX2(write);
 7034   src1   : EX1(read);
 7035   INS01  : ISS;
 7036   ALU    : EX2;
 7037 %}
 7038 
 7039 // Integer ALU immediate operation (no source operands)
 7040 // Eg.  MOV     x0, #N
 7041 pipe_class ialu_imm(iRegI dst)
 7042 %{
 7043   single_instruction;
 7044   dst    : EX1(write);
 7045   INS01  : ISS;
 7046   ALU    : EX1;
 7047 %}
 7048 
 7049 //------- Compare operation -------------------------------
 7050 
 7051 // Compare reg-reg
 7052 // Eg.  CMP     x0, x1
 7053 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 7054 %{
 7055   single_instruction;
 7056 //  fixed_latency(16);
 7057   cr     : EX2(write);
 7058   op1    : EX1(read);
 7059   op2    : EX1(read);
 7060   INS01  : ISS;
 7061   ALU    : EX2;
 7062 %}
 7063 
 7064 // Compare reg-reg
 7065 // Eg.  CMP     x0, #N
 7066 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 7067 %{
 7068   single_instruction;
 7069 //  fixed_latency(16);
 7070   cr     : EX2(write);
 7071   op1    : EX1(read);
 7072   INS01  : ISS;
 7073   ALU    : EX2;
 7074 %}
 7075 
 7076 //------- Conditional instructions ------------------------
 7077 
 7078 // Conditional no operands
 7079 // Eg.  CSINC   x0, zr, zr, <cond>
 7080 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 7081 %{
 7082   single_instruction;
 7083   cr     : EX1(read);
 7084   dst    : EX2(write);
 7085   INS01  : ISS;
 7086   ALU    : EX2;
 7087 %}
 7088 
 7089 // Conditional 2 operand
 7090 // EG.  CSEL    X0, X1, X2, <cond>
 7091 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 7092 %{
 7093   single_instruction;
 7094   cr     : EX1(read);
 7095   src1   : EX1(read);
 7096   src2   : EX1(read);
 7097   dst    : EX2(write);
 7098   INS01  : ISS;
 7099   ALU    : EX2;
 7100 %}
 7101 
 7102 // Conditional 2 operand
 7103 // EG.  CSEL    X0, X1, X2, <cond>
 7104 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 7105 %{
 7106   single_instruction;
 7107   cr     : EX1(read);
 7108   src    : EX1(read);
 7109   dst    : EX2(write);
 7110   INS01  : ISS;
 7111   ALU    : EX2;
 7112 %}
 7113 
 7114 //------- Multiply pipeline operations --------------------
 7115 
 7116 // Multiply reg-reg
 7117 // Eg.  MUL     w0, w1, w2
 7118 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7119 %{
 7120   single_instruction;
 7121   dst    : WR(write);
 7122   src1   : ISS(read);
 7123   src2   : ISS(read);
 7124   INS01  : ISS;
 7125   MAC    : WR;
 7126 %}
 7127 
 7128 // Multiply accumulate
 7129 // Eg.  MADD    w0, w1, w2, w3
 7130 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7131 %{
 7132   single_instruction;
 7133   dst    : WR(write);
 7134   src1   : ISS(read);
 7135   src2   : ISS(read);
 7136   src3   : ISS(read);
 7137   INS01  : ISS;
 7138   MAC    : WR;
 7139 %}
 7140 
 7141 // Eg.  MUL     w0, w1, w2
 7142 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7143 %{
 7144   single_instruction;
 7145   fixed_latency(3); // Maximum latency for 64 bit mul
 7146   dst    : WR(write);
 7147   src1   : ISS(read);
 7148   src2   : ISS(read);
 7149   INS01  : ISS;
 7150   MAC    : WR;
 7151 %}
 7152 
 7153 // Multiply accumulate
 7154 // Eg.  MADD    w0, w1, w2, w3
 7155 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7156 %{
 7157   single_instruction;
 7158   fixed_latency(3); // Maximum latency for 64 bit mul
 7159   dst    : WR(write);
 7160   src1   : ISS(read);
 7161   src2   : ISS(read);
 7162   src3   : ISS(read);
 7163   INS01  : ISS;
 7164   MAC    : WR;
 7165 %}
 7166 
 7167 //------- Divide pipeline operations --------------------
 7168 
 7169 // Eg.  SDIV    w0, w1, w2
 7170 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7171 %{
 7172   single_instruction;
 7173   fixed_latency(8); // Maximum latency for 32 bit divide
 7174   dst    : WR(write);
 7175   src1   : ISS(read);
 7176   src2   : ISS(read);
 7177   INS0   : ISS; // Can only dual issue as instruction 0
 7178   DIV    : WR;
 7179 %}
 7180 
 7181 // Eg.  SDIV    x0, x1, x2
 7182 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7183 %{
 7184   single_instruction;
 7185   fixed_latency(16); // Maximum latency for 64 bit divide
 7186   dst    : WR(write);
 7187   src1   : ISS(read);
 7188   src2   : ISS(read);
 7189   INS0   : ISS; // Can only dual issue as instruction 0
 7190   DIV    : WR;
 7191 %}
 7192 
 7193 //------- Load pipeline operations ------------------------
 7194 
 7195 // Load - prefetch
 7196 // Eg.  PFRM    <mem>
 7197 pipe_class iload_prefetch(memory mem)
 7198 %{
 7199   single_instruction;
 7200   mem    : ISS(read);
 7201   INS01  : ISS;
 7202   LDST   : WR;
 7203 %}
 7204 
 7205 // Load - reg, mem
 7206 // Eg.  LDR     x0, <mem>
 7207 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7208 %{
 7209   single_instruction;
 7210   dst    : WR(write);
 7211   mem    : ISS(read);
 7212   INS01  : ISS;
 7213   LDST   : WR;
 7214 %}
 7215 
 7216 // Load - reg, reg
 7217 // Eg.  LDR     x0, [sp, x1]
 7218 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7219 %{
 7220   single_instruction;
 7221   dst    : WR(write);
 7222   src    : ISS(read);
 7223   INS01  : ISS;
 7224   LDST   : WR;
 7225 %}
 7226 
 7227 //------- Store pipeline operations -----------------------
 7228 
 7229 // Store - zr, mem
 7230 // Eg.  STR     zr, <mem>
 7231 pipe_class istore_mem(memory mem)
 7232 %{
 7233   single_instruction;
 7234   mem    : ISS(read);
 7235   INS01  : ISS;
 7236   LDST   : WR;
 7237 %}
 7238 
 7239 // Store - reg, mem
 7240 // Eg.  STR     x0, <mem>
 7241 pipe_class istore_reg_mem(iRegI src, memory mem)
 7242 %{
 7243   single_instruction;
 7244   mem    : ISS(read);
 7245   src    : EX2(read);
 7246   INS01  : ISS;
 7247   LDST   : WR;
 7248 %}
 7249 
 7250 // Store - reg, reg
 7251 // Eg. STR      x0, [sp, x1]
 7252 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7253 %{
 7254   single_instruction;
 7255   dst    : ISS(read);
 7256   src    : EX2(read);
 7257   INS01  : ISS;
 7258   LDST   : WR;
 7259 %}
 7260 
 7261 //------- Store pipeline operations -----------------------
 7262 
 7263 // Branch
 7264 pipe_class pipe_branch()
 7265 %{
 7266   single_instruction;
 7267   INS01  : ISS;
 7268   BRANCH : EX1;
 7269 %}
 7270 
 7271 // Conditional branch
 7272 pipe_class pipe_branch_cond(rFlagsReg cr)
 7273 %{
 7274   single_instruction;
 7275   cr     : EX1(read);
 7276   INS01  : ISS;
 7277   BRANCH : EX1;
 7278 %}
 7279 
 7280 // Compare & Branch
 7281 // EG.  CBZ/CBNZ
 7282 pipe_class pipe_cmp_branch(iRegI op1)
 7283 %{
 7284   single_instruction;
 7285   op1    : EX1(read);
 7286   INS01  : ISS;
 7287   BRANCH : EX1;
 7288 %}
 7289 
 7290 //------- Synchronisation operations ----------------------
 7291 
 7292 // Any operation requiring serialization.
 7293 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7294 pipe_class pipe_serial()
 7295 %{
 7296   single_instruction;
 7297   force_serialization;
 7298   fixed_latency(16);
 7299   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7300   LDST   : WR;
 7301 %}
 7302 
 7303 // Generic big/slow expanded idiom - also serialized
 7304 pipe_class pipe_slow()
 7305 %{
 7306   instruction_count(10);
 7307   multiple_bundles;
 7308   force_serialization;
 7309   fixed_latency(16);
 7310   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7311   LDST   : WR;
 7312 %}
 7313 
 7314 // Empty pipeline class
 7315 pipe_class pipe_class_empty()
 7316 %{
 7317   single_instruction;
 7318   fixed_latency(0);
 7319 %}
 7320 
 7321 // Default pipeline class.
 7322 pipe_class pipe_class_default()
 7323 %{
 7324   single_instruction;
 7325   fixed_latency(2);
 7326 %}
 7327 
 7328 // Pipeline class for compares.
 7329 pipe_class pipe_class_compare()
 7330 %{
 7331   single_instruction;
 7332   fixed_latency(16);
 7333 %}
 7334 
 7335 // Pipeline class for memory operations.
 7336 pipe_class pipe_class_memory()
 7337 %{
 7338   single_instruction;
 7339   fixed_latency(16);
 7340 %}
 7341 
 7342 // Pipeline class for call.
 7343 pipe_class pipe_class_call()
 7344 %{
 7345   single_instruction;
 7346   fixed_latency(100);
 7347 %}
 7348 
 7349 // Define the class for the Nop node.
 7350 define %{
 7351    MachNop = pipe_class_empty;
 7352 %}
 7353 
 7354 %}
 7355 //----------INSTRUCTIONS-------------------------------------------------------
 7356 //
 7357 // match      -- States which machine-independent subtree may be replaced
 7358 //               by this instruction.
 7359 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7360 //               selection to identify a minimum cost tree of machine
 7361 //               instructions that matches a tree of machine-independent
 7362 //               instructions.
 7363 // format     -- A string providing the disassembly for this instruction.
 7364 //               The value of an instruction's operand may be inserted
 7365 //               by referring to it with a '$' prefix.
 7366 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7367 //               to within an encode class as $primary, $secondary, and $tertiary
 7368 //               rrspectively.  The primary opcode is commonly used to
 7369 //               indicate the type of machine instruction, while secondary
 7370 //               and tertiary are often used for prefix options or addressing
 7371 //               modes.
 7372 // ins_encode -- A list of encode classes with parameters. The encode class
 7373 //               name must have been defined in an 'enc_class' specification
 7374 //               in the encode section of the architecture description.
 7375 
 7376 // ============================================================================
 7377 // Memory (Load/Store) Instructions
 7378 
 7379 // Load Instructions
 7380 
 7381 // Load Byte (8 bit signed)
 7382 instruct loadB(iRegINoSp dst, memory1 mem)
 7383 %{
 7384   match(Set dst (LoadB mem));
 7385   predicate(!needs_acquiring_load(n));
 7386 
 7387   ins_cost(4 * INSN_COST);
 7388   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7389 
 7390   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7391 
 7392   ins_pipe(iload_reg_mem);
 7393 %}
 7394 
 7395 // Load Byte (8 bit signed) into long
 7396 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7397 %{
 7398   match(Set dst (ConvI2L (LoadB mem)));
 7399   predicate(!needs_acquiring_load(n->in(1)));
 7400 
 7401   ins_cost(4 * INSN_COST);
 7402   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7403 
 7404   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7405 
 7406   ins_pipe(iload_reg_mem);
 7407 %}
 7408 
 7409 // Load Byte (8 bit unsigned)
 7410 instruct loadUB(iRegINoSp dst, memory1 mem)
 7411 %{
 7412   match(Set dst (LoadUB mem));
 7413   predicate(!needs_acquiring_load(n));
 7414 
 7415   ins_cost(4 * INSN_COST);
 7416   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7417 
 7418   ins_encode(aarch64_enc_ldrb(dst, mem));
 7419 
 7420   ins_pipe(iload_reg_mem);
 7421 %}
 7422 
 7423 // Load Byte (8 bit unsigned) into long
 7424 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7425 %{
 7426   match(Set dst (ConvI2L (LoadUB mem)));
 7427   predicate(!needs_acquiring_load(n->in(1)));
 7428 
 7429   ins_cost(4 * INSN_COST);
 7430   format %{ "ldrb  $dst, $mem\t# byte" %}
 7431 
 7432   ins_encode(aarch64_enc_ldrb(dst, mem));
 7433 
 7434   ins_pipe(iload_reg_mem);
 7435 %}
 7436 
 7437 // Load Short (16 bit signed)
 7438 instruct loadS(iRegINoSp dst, memory2 mem)
 7439 %{
 7440   match(Set dst (LoadS mem));
 7441   predicate(!needs_acquiring_load(n));
 7442 
 7443   ins_cost(4 * INSN_COST);
 7444   format %{ "ldrshw  $dst, $mem\t# short" %}
 7445 
 7446   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7447 
 7448   ins_pipe(iload_reg_mem);
 7449 %}
 7450 
 7451 // Load Short (16 bit signed) into long
 7452 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7453 %{
 7454   match(Set dst (ConvI2L (LoadS mem)));
 7455   predicate(!needs_acquiring_load(n->in(1)));
 7456 
 7457   ins_cost(4 * INSN_COST);
 7458   format %{ "ldrsh  $dst, $mem\t# short" %}
 7459 
 7460   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7461 
 7462   ins_pipe(iload_reg_mem);
 7463 %}
 7464 
 7465 // Load Char (16 bit unsigned)
 7466 instruct loadUS(iRegINoSp dst, memory2 mem)
 7467 %{
 7468   match(Set dst (LoadUS mem));
 7469   predicate(!needs_acquiring_load(n));
 7470 
 7471   ins_cost(4 * INSN_COST);
 7472   format %{ "ldrh  $dst, $mem\t# short" %}
 7473 
 7474   ins_encode(aarch64_enc_ldrh(dst, mem));
 7475 
 7476   ins_pipe(iload_reg_mem);
 7477 %}
 7478 
 7479 // Load Short/Char (16 bit unsigned) into long
 7480 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7481 %{
 7482   match(Set dst (ConvI2L (LoadUS mem)));
 7483   predicate(!needs_acquiring_load(n->in(1)));
 7484 
 7485   ins_cost(4 * INSN_COST);
 7486   format %{ "ldrh  $dst, $mem\t# short" %}
 7487 
 7488   ins_encode(aarch64_enc_ldrh(dst, mem));
 7489 
 7490   ins_pipe(iload_reg_mem);
 7491 %}
 7492 
 7493 // Load Integer (32 bit signed)
 7494 instruct loadI(iRegINoSp dst, memory4 mem)
 7495 %{
 7496   match(Set dst (LoadI mem));
 7497   predicate(!needs_acquiring_load(n));
 7498 
 7499   ins_cost(4 * INSN_COST);
 7500   format %{ "ldrw  $dst, $mem\t# int" %}
 7501 
 7502   ins_encode(aarch64_enc_ldrw(dst, mem));
 7503 
 7504   ins_pipe(iload_reg_mem);
 7505 %}
 7506 
 7507 // Load Integer (32 bit signed) into long
 7508 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7509 %{
 7510   match(Set dst (ConvI2L (LoadI mem)));
 7511   predicate(!needs_acquiring_load(n->in(1)));
 7512 
 7513   ins_cost(4 * INSN_COST);
 7514   format %{ "ldrsw  $dst, $mem\t# int" %}
 7515 
 7516   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7517 
 7518   ins_pipe(iload_reg_mem);
 7519 %}
 7520 
 7521 // Load Integer (32 bit unsigned) into long
 7522 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7523 %{
 7524   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7525   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7526 
 7527   ins_cost(4 * INSN_COST);
 7528   format %{ "ldrw  $dst, $mem\t# int" %}
 7529 
 7530   ins_encode(aarch64_enc_ldrw(dst, mem));
 7531 
 7532   ins_pipe(iload_reg_mem);
 7533 %}
 7534 
 7535 // Load Long (64 bit signed)
 7536 instruct loadL(iRegLNoSp dst, memory8 mem)
 7537 %{
 7538   match(Set dst (LoadL mem));
 7539   predicate(!needs_acquiring_load(n));
 7540 
 7541   ins_cost(4 * INSN_COST);
 7542   format %{ "ldr  $dst, $mem\t# int" %}
 7543 
 7544   ins_encode(aarch64_enc_ldr(dst, mem));
 7545 
 7546   ins_pipe(iload_reg_mem);
 7547 %}
 7548 
 7549 // Load Range
 7550 instruct loadRange(iRegINoSp dst, memory4 mem)
 7551 %{
 7552   match(Set dst (LoadRange mem));
 7553 
 7554   ins_cost(4 * INSN_COST);
 7555   format %{ "ldrw  $dst, $mem\t# range" %}
 7556 
 7557   ins_encode(aarch64_enc_ldrw(dst, mem));
 7558 
 7559   ins_pipe(iload_reg_mem);
 7560 %}
 7561 
 7562 // Load Pointer
 7563 instruct loadP(iRegPNoSp dst, memory8 mem)
 7564 %{
 7565   match(Set dst (LoadP mem));
 7566   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7567 
 7568   ins_cost(4 * INSN_COST);
 7569   format %{ "ldr  $dst, $mem\t# ptr" %}
 7570 
 7571   ins_encode(aarch64_enc_ldr(dst, mem));
 7572 
 7573   ins_pipe(iload_reg_mem);
 7574 %}
 7575 
 7576 // Load Compressed Pointer
 7577 instruct loadN(iRegNNoSp dst, memory4 mem)
 7578 %{
 7579   match(Set dst (LoadN mem));
 7580   predicate(!needs_acquiring_load(n));
 7581 
 7582   ins_cost(4 * INSN_COST);
 7583   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7584 
 7585   ins_encode(aarch64_enc_ldrw(dst, mem));
 7586 
 7587   ins_pipe(iload_reg_mem);
 7588 %}
 7589 
 7590 // Load Klass Pointer
 7591 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7592 %{
 7593   match(Set dst (LoadKlass mem));
 7594   predicate(!needs_acquiring_load(n));
 7595 
 7596   ins_cost(4 * INSN_COST);
 7597   format %{ "ldr  $dst, $mem\t# class" %}
 7598 
 7599   ins_encode(aarch64_enc_ldr(dst, mem));
 7600 
 7601   ins_pipe(iload_reg_mem);
 7602 %}
 7603 
 7604 // Load Narrow Klass Pointer
 7605 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7606 %{
 7607   match(Set dst (LoadNKlass mem));
 7608   predicate(!needs_acquiring_load(n));
 7609 
 7610   ins_cost(4 * INSN_COST);
 7611   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7612 
 7613   ins_encode(aarch64_enc_ldrw(dst, mem));
 7614 
 7615   ins_pipe(iload_reg_mem);
 7616 %}
 7617 
 7618 // Load Float
 7619 instruct loadF(vRegF dst, memory4 mem)
 7620 %{
 7621   match(Set dst (LoadF mem));
 7622   predicate(!needs_acquiring_load(n));
 7623 
 7624   ins_cost(4 * INSN_COST);
 7625   format %{ "ldrs  $dst, $mem\t# float" %}
 7626 
 7627   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7628 
 7629   ins_pipe(pipe_class_memory);
 7630 %}
 7631 
 7632 // Load Double
 7633 instruct loadD(vRegD dst, memory8 mem)
 7634 %{
 7635   match(Set dst (LoadD mem));
 7636   predicate(!needs_acquiring_load(n));
 7637 
 7638   ins_cost(4 * INSN_COST);
 7639   format %{ "ldrd  $dst, $mem\t# double" %}
 7640 
 7641   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7642 
 7643   ins_pipe(pipe_class_memory);
 7644 %}
 7645 
 7646 
 7647 // Load Int Constant
 7648 instruct loadConI(iRegINoSp dst, immI src)
 7649 %{
 7650   match(Set dst src);
 7651 
 7652   ins_cost(INSN_COST);
 7653   format %{ "mov $dst, $src\t# int" %}
 7654 
 7655   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7656 
 7657   ins_pipe(ialu_imm);
 7658 %}
 7659 
 7660 // Load Long Constant
 7661 instruct loadConL(iRegLNoSp dst, immL src)
 7662 %{
 7663   match(Set dst src);
 7664 
 7665   ins_cost(INSN_COST);
 7666   format %{ "mov $dst, $src\t# long" %}
 7667 
 7668   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7669 
 7670   ins_pipe(ialu_imm);
 7671 %}
 7672 
 7673 // Load Pointer Constant
 7674 
 7675 instruct loadConP(iRegPNoSp dst, immP con)
 7676 %{
 7677   match(Set dst con);
 7678 
 7679   ins_cost(INSN_COST * 4);
 7680   format %{
 7681     "mov  $dst, $con\t# ptr\n\t"
 7682   %}
 7683 
 7684   ins_encode(aarch64_enc_mov_p(dst, con));
 7685 
 7686   ins_pipe(ialu_imm);
 7687 %}
 7688 
 7689 // Load Null Pointer Constant
 7690 
 7691 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7692 %{
 7693   match(Set dst con);
 7694 
 7695   ins_cost(INSN_COST);
 7696   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7697 
 7698   ins_encode(aarch64_enc_mov_p0(dst, con));
 7699 
 7700   ins_pipe(ialu_imm);
 7701 %}
 7702 
 7703 // Load Pointer Constant One
 7704 
 7705 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7706 %{
 7707   match(Set dst con);
 7708 
 7709   ins_cost(INSN_COST);
 7710   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7711 
 7712   ins_encode(aarch64_enc_mov_p1(dst, con));
 7713 
 7714   ins_pipe(ialu_imm);
 7715 %}
 7716 
 7717 // Load Byte Map Base Constant
 7718 
 7719 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7720 %{
 7721   match(Set dst con);
 7722 
 7723   ins_cost(INSN_COST);
 7724   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7725 
 7726   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7727 
 7728   ins_pipe(ialu_imm);
 7729 %}
 7730 
 7731 // Load Narrow Pointer Constant
 7732 
 7733 instruct loadConN(iRegNNoSp dst, immN con)
 7734 %{
 7735   match(Set dst con);
 7736 
 7737   ins_cost(INSN_COST * 4);
 7738   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7739 
 7740   ins_encode(aarch64_enc_mov_n(dst, con));
 7741 
 7742   ins_pipe(ialu_imm);
 7743 %}
 7744 
 7745 // Load Narrow Null Pointer Constant
 7746 
 7747 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7748 %{
 7749   match(Set dst con);
 7750 
 7751   ins_cost(INSN_COST);
 7752   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7753 
 7754   ins_encode(aarch64_enc_mov_n0(dst, con));
 7755 
 7756   ins_pipe(ialu_imm);
 7757 %}
 7758 
 7759 // Load Narrow Klass Constant
 7760 
 7761 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7762 %{
 7763   match(Set dst con);
 7764 
 7765   ins_cost(INSN_COST);
 7766   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7767 
 7768   ins_encode(aarch64_enc_mov_nk(dst, con));
 7769 
 7770   ins_pipe(ialu_imm);
 7771 %}
 7772 
 7773 // Load Packed Float Constant
 7774 
 7775 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7776   match(Set dst con);
 7777   ins_cost(INSN_COST * 4);
 7778   format %{ "fmovs  $dst, $con"%}
 7779   ins_encode %{
 7780     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7781   %}
 7782 
 7783   ins_pipe(fp_imm_s);
 7784 %}
 7785 
 7786 // Load Float Constant
 7787 
 7788 instruct loadConF(vRegF dst, immF con) %{
 7789   match(Set dst con);
 7790 
 7791   ins_cost(INSN_COST * 4);
 7792 
 7793   format %{
 7794     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7795   %}
 7796 
 7797   ins_encode %{
 7798     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7799   %}
 7800 
 7801   ins_pipe(fp_load_constant_s);
 7802 %}
 7803 
 7804 // Load Packed Double Constant
 7805 
 7806 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7807   match(Set dst con);
 7808   ins_cost(INSN_COST);
 7809   format %{ "fmovd  $dst, $con"%}
 7810   ins_encode %{
 7811     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7812   %}
 7813 
 7814   ins_pipe(fp_imm_d);
 7815 %}
 7816 
 7817 // Load Double Constant
 7818 
 7819 instruct loadConD(vRegD dst, immD con) %{
 7820   match(Set dst con);
 7821 
 7822   ins_cost(INSN_COST * 5);
 7823   format %{
 7824     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7825   %}
 7826 
 7827   ins_encode %{
 7828     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7829   %}
 7830 
 7831   ins_pipe(fp_load_constant_d);
 7832 %}
 7833 
 7834 // Store Instructions
 7835 
 7836 // Store CMS card-mark Immediate
 7837 instruct storeimmCM0(immI0 zero, memory1 mem)
 7838 %{
 7839   match(Set mem (StoreCM mem zero));
 7840 
 7841   ins_cost(INSN_COST);
 7842   format %{ "storestore (elided)\n\t"
 7843             "strb zr, $mem\t# byte" %}
 7844 
 7845   ins_encode(aarch64_enc_strb0(mem));
 7846 
 7847   ins_pipe(istore_mem);
 7848 %}
 7849 
 7850 // Store CMS card-mark Immediate with intervening StoreStore
 7851 // needed when using CMS with no conditional card marking
 7852 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7853 %{
 7854   match(Set mem (StoreCM mem zero));
 7855 
 7856   ins_cost(INSN_COST * 2);
 7857   format %{ "storestore\n\t"
 7858             "dmb ishst"
 7859             "\n\tstrb zr, $mem\t# byte" %}
 7860 
 7861   ins_encode(aarch64_enc_strb0_ordered(mem));
 7862 
 7863   ins_pipe(istore_mem);
 7864 %}
 7865 
 7866 // Store Byte
 7867 instruct storeB(iRegIorL2I src, memory1 mem)
 7868 %{
 7869   match(Set mem (StoreB mem src));
 7870   predicate(!needs_releasing_store(n));
 7871 
 7872   ins_cost(INSN_COST);
 7873   format %{ "strb  $src, $mem\t# byte" %}
 7874 
 7875   ins_encode(aarch64_enc_strb(src, mem));
 7876 
 7877   ins_pipe(istore_reg_mem);
 7878 %}
 7879 
 7880 
 7881 instruct storeimmB0(immI0 zero, memory1 mem)
 7882 %{
 7883   match(Set mem (StoreB mem zero));
 7884   predicate(!needs_releasing_store(n));
 7885 
 7886   ins_cost(INSN_COST);
 7887   format %{ "strb rscractch2, $mem\t# byte" %}
 7888 
 7889   ins_encode(aarch64_enc_strb0(mem));
 7890 
 7891   ins_pipe(istore_mem);
 7892 %}
 7893 
 7894 // Store Char/Short
 7895 instruct storeC(iRegIorL2I src, memory2 mem)
 7896 %{
 7897   match(Set mem (StoreC mem src));
 7898   predicate(!needs_releasing_store(n));
 7899 
 7900   ins_cost(INSN_COST);
 7901   format %{ "strh  $src, $mem\t# short" %}
 7902 
 7903   ins_encode(aarch64_enc_strh(src, mem));
 7904 
 7905   ins_pipe(istore_reg_mem);
 7906 %}
 7907 
 7908 instruct storeimmC0(immI0 zero, memory2 mem)
 7909 %{
 7910   match(Set mem (StoreC mem zero));
 7911   predicate(!needs_releasing_store(n));
 7912 
 7913   ins_cost(INSN_COST);
 7914   format %{ "strh  zr, $mem\t# short" %}
 7915 
 7916   ins_encode(aarch64_enc_strh0(mem));
 7917 
 7918   ins_pipe(istore_mem);
 7919 %}
 7920 
 7921 // Store Integer
 7922 
 7923 instruct storeI(iRegIorL2I src, memory4 mem)
 7924 %{
 7925   match(Set mem(StoreI mem src));
 7926   predicate(!needs_releasing_store(n));
 7927 
 7928   ins_cost(INSN_COST);
 7929   format %{ "strw  $src, $mem\t# int" %}
 7930 
 7931   ins_encode(aarch64_enc_strw(src, mem));
 7932 
 7933   ins_pipe(istore_reg_mem);
 7934 %}
 7935 
 7936 instruct storeimmI0(immI0 zero, memory4 mem)
 7937 %{
 7938   match(Set mem(StoreI mem zero));
 7939   predicate(!needs_releasing_store(n));
 7940 
 7941   ins_cost(INSN_COST);
 7942   format %{ "strw  zr, $mem\t# int" %}
 7943 
 7944   ins_encode(aarch64_enc_strw0(mem));
 7945 
 7946   ins_pipe(istore_mem);
 7947 %}
 7948 
 7949 // Store Long (64 bit signed)
 7950 instruct storeL(iRegL src, memory8 mem)
 7951 %{
 7952   match(Set mem (StoreL mem src));
 7953   predicate(!needs_releasing_store(n));
 7954 
 7955   ins_cost(INSN_COST);
 7956   format %{ "str  $src, $mem\t# int" %}
 7957 
 7958   ins_encode(aarch64_enc_str(src, mem));
 7959 
 7960   ins_pipe(istore_reg_mem);
 7961 %}
 7962 
 7963 // Store Long (64 bit signed)
 7964 instruct storeimmL0(immL0 zero, memory8 mem)
 7965 %{
 7966   match(Set mem (StoreL mem zero));
 7967   predicate(!needs_releasing_store(n));
 7968 
 7969   ins_cost(INSN_COST);
 7970   format %{ "str  zr, $mem\t# int" %}
 7971 
 7972   ins_encode(aarch64_enc_str0(mem));
 7973 
 7974   ins_pipe(istore_mem);
 7975 %}
 7976 
 7977 // Store Pointer
 7978 instruct storeP(iRegP src, memory8 mem)
 7979 %{
 7980   match(Set mem (StoreP mem src));
 7981   predicate(!needs_releasing_store(n));
 7982 
 7983   ins_cost(INSN_COST);
 7984   format %{ "str  $src, $mem\t# ptr" %}
 7985 
 7986   ins_encode(aarch64_enc_str(src, mem));
 7987 
 7988   ins_pipe(istore_reg_mem);
 7989 %}
 7990 
 7991 // Store Pointer
 7992 instruct storeimmP0(immP0 zero, memory8 mem)
 7993 %{
 7994   match(Set mem (StoreP mem zero));
 7995   predicate(!needs_releasing_store(n));
 7996 
 7997   ins_cost(INSN_COST);
 7998   format %{ "str zr, $mem\t# ptr" %}
 7999 
 8000   ins_encode(aarch64_enc_str0(mem));
 8001 
 8002   ins_pipe(istore_mem);
 8003 %}
 8004 
 8005 // Store Compressed Pointer
 8006 instruct storeN(iRegN src, memory4 mem)
 8007 %{
 8008   match(Set mem (StoreN mem src));
 8009   predicate(!needs_releasing_store(n));
 8010 
 8011   ins_cost(INSN_COST);
 8012   format %{ "strw  $src, $mem\t# compressed ptr" %}
 8013 
 8014   ins_encode(aarch64_enc_strw(src, mem));
 8015 
 8016   ins_pipe(istore_reg_mem);
 8017 %}
 8018 
 8019 instruct storeImmN0(immN0 zero, memory4 mem)
 8020 %{
 8021   match(Set mem (StoreN mem zero));
 8022   predicate(!needs_releasing_store(n));
 8023 
 8024   ins_cost(INSN_COST);
 8025   format %{ "strw  zr, $mem\t# compressed ptr" %}
 8026 
 8027   ins_encode(aarch64_enc_strw0(mem));
 8028 
 8029   ins_pipe(istore_mem);
 8030 %}
 8031 
 8032 // Store Float
 8033 instruct storeF(vRegF src, memory4 mem)
 8034 %{
 8035   match(Set mem (StoreF mem src));
 8036   predicate(!needs_releasing_store(n));
 8037 
 8038   ins_cost(INSN_COST);
 8039   format %{ "strs  $src, $mem\t# float" %}
 8040 
 8041   ins_encode( aarch64_enc_strs(src, mem) );
 8042 
 8043   ins_pipe(pipe_class_memory);
 8044 %}
 8045 
 8046 // TODO
 8047 // implement storeImmF0 and storeFImmPacked
 8048 
 8049 // Store Double
 8050 instruct storeD(vRegD src, memory8 mem)
 8051 %{
 8052   match(Set mem (StoreD mem src));
 8053   predicate(!needs_releasing_store(n));
 8054 
 8055   ins_cost(INSN_COST);
 8056   format %{ "strd  $src, $mem\t# double" %}
 8057 
 8058   ins_encode( aarch64_enc_strd(src, mem) );
 8059 
 8060   ins_pipe(pipe_class_memory);
 8061 %}
 8062 
 8063 // Store Compressed Klass Pointer
 8064 instruct storeNKlass(iRegN src, memory4 mem)
 8065 %{
 8066   predicate(!needs_releasing_store(n));
 8067   match(Set mem (StoreNKlass mem src));
 8068 
 8069   ins_cost(INSN_COST);
 8070   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 8071 
 8072   ins_encode(aarch64_enc_strw(src, mem));
 8073 
 8074   ins_pipe(istore_reg_mem);
 8075 %}
 8076 
 8077 // TODO
 8078 // implement storeImmD0 and storeDImmPacked
 8079 
 8080 // prefetch instructions
 8081 // Must be safe to execute with invalid address (cannot fault).
 8082 
 8083 instruct prefetchalloc( memory8 mem ) %{
 8084   match(PrefetchAllocation mem);
 8085 
 8086   ins_cost(INSN_COST);
 8087   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 8088 
 8089   ins_encode( aarch64_enc_prefetchw(mem) );
 8090 
 8091   ins_pipe(iload_prefetch);
 8092 %}
 8093 
 8094 //  ---------------- volatile loads and stores ----------------
 8095 
 8096 // Load Byte (8 bit signed)
 8097 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8098 %{
 8099   match(Set dst (LoadB mem));
 8100 
 8101   ins_cost(VOLATILE_REF_COST);
 8102   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8103 
 8104   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8105 
 8106   ins_pipe(pipe_serial);
 8107 %}
 8108 
 8109 // Load Byte (8 bit signed) into long
 8110 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8111 %{
 8112   match(Set dst (ConvI2L (LoadB mem)));
 8113 
 8114   ins_cost(VOLATILE_REF_COST);
 8115   format %{ "ldarsb  $dst, $mem\t# byte" %}
 8116 
 8117   ins_encode(aarch64_enc_ldarsb(dst, mem));
 8118 
 8119   ins_pipe(pipe_serial);
 8120 %}
 8121 
 8122 // Load Byte (8 bit unsigned)
 8123 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8124 %{
 8125   match(Set dst (LoadUB mem));
 8126 
 8127   ins_cost(VOLATILE_REF_COST);
 8128   format %{ "ldarb  $dst, $mem\t# byte" %}
 8129 
 8130   ins_encode(aarch64_enc_ldarb(dst, mem));
 8131 
 8132   ins_pipe(pipe_serial);
 8133 %}
 8134 
 8135 // Load Byte (8 bit unsigned) into long
 8136 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8137 %{
 8138   match(Set dst (ConvI2L (LoadUB mem)));
 8139 
 8140   ins_cost(VOLATILE_REF_COST);
 8141   format %{ "ldarb  $dst, $mem\t# byte" %}
 8142 
 8143   ins_encode(aarch64_enc_ldarb(dst, mem));
 8144 
 8145   ins_pipe(pipe_serial);
 8146 %}
 8147 
 8148 // Load Short (16 bit signed)
 8149 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8150 %{
 8151   match(Set dst (LoadS mem));
 8152 
 8153   ins_cost(VOLATILE_REF_COST);
 8154   format %{ "ldarshw  $dst, $mem\t# short" %}
 8155 
 8156   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8157 
 8158   ins_pipe(pipe_serial);
 8159 %}
 8160 
 8161 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8162 %{
 8163   match(Set dst (LoadUS mem));
 8164 
 8165   ins_cost(VOLATILE_REF_COST);
 8166   format %{ "ldarhw  $dst, $mem\t# short" %}
 8167 
 8168   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8169 
 8170   ins_pipe(pipe_serial);
 8171 %}
 8172 
 8173 // Load Short/Char (16 bit unsigned) into long
 8174 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8175 %{
 8176   match(Set dst (ConvI2L (LoadUS mem)));
 8177 
 8178   ins_cost(VOLATILE_REF_COST);
 8179   format %{ "ldarh  $dst, $mem\t# short" %}
 8180 
 8181   ins_encode(aarch64_enc_ldarh(dst, mem));
 8182 
 8183   ins_pipe(pipe_serial);
 8184 %}
 8185 
 8186 // Load Short/Char (16 bit signed) into long
 8187 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8188 %{
 8189   match(Set dst (ConvI2L (LoadS mem)));
 8190 
 8191   ins_cost(VOLATILE_REF_COST);
 8192   format %{ "ldarh  $dst, $mem\t# short" %}
 8193 
 8194   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8195 
 8196   ins_pipe(pipe_serial);
 8197 %}
 8198 
 8199 // Load Integer (32 bit signed)
 8200 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8201 %{
 8202   match(Set dst (LoadI mem));
 8203 
 8204   ins_cost(VOLATILE_REF_COST);
 8205   format %{ "ldarw  $dst, $mem\t# int" %}
 8206 
 8207   ins_encode(aarch64_enc_ldarw(dst, mem));
 8208 
 8209   ins_pipe(pipe_serial);
 8210 %}
 8211 
 8212 // Load Integer (32 bit unsigned) into long
 8213 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8214 %{
 8215   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8216 
 8217   ins_cost(VOLATILE_REF_COST);
 8218   format %{ "ldarw  $dst, $mem\t# int" %}
 8219 
 8220   ins_encode(aarch64_enc_ldarw(dst, mem));
 8221 
 8222   ins_pipe(pipe_serial);
 8223 %}
 8224 
 8225 // Load Long (64 bit signed)
 8226 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8227 %{
 8228   match(Set dst (LoadL mem));
 8229 
 8230   ins_cost(VOLATILE_REF_COST);
 8231   format %{ "ldar  $dst, $mem\t# int" %}
 8232 
 8233   ins_encode(aarch64_enc_ldar(dst, mem));
 8234 
 8235   ins_pipe(pipe_serial);
 8236 %}
 8237 
 8238 // Load Pointer
 8239 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8240 %{
 8241   match(Set dst (LoadP mem));
 8242   predicate(n->as_Load()->barrier_data() == 0);
 8243 
 8244   ins_cost(VOLATILE_REF_COST);
 8245   format %{ "ldar  $dst, $mem\t# ptr" %}
 8246 
 8247   ins_encode(aarch64_enc_ldar(dst, mem));
 8248 
 8249   ins_pipe(pipe_serial);
 8250 %}
 8251 
 8252 // Load Compressed Pointer
 8253 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8254 %{
 8255   match(Set dst (LoadN mem));
 8256 
 8257   ins_cost(VOLATILE_REF_COST);
 8258   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8259 
 8260   ins_encode(aarch64_enc_ldarw(dst, mem));
 8261 
 8262   ins_pipe(pipe_serial);
 8263 %}
 8264 
 8265 // Load Float
 8266 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8267 %{
 8268   match(Set dst (LoadF mem));
 8269 
 8270   ins_cost(VOLATILE_REF_COST);
 8271   format %{ "ldars  $dst, $mem\t# float" %}
 8272 
 8273   ins_encode( aarch64_enc_fldars(dst, mem) );
 8274 
 8275   ins_pipe(pipe_serial);
 8276 %}
 8277 
 8278 // Load Double
 8279 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8280 %{
 8281   match(Set dst (LoadD mem));
 8282 
 8283   ins_cost(VOLATILE_REF_COST);
 8284   format %{ "ldard  $dst, $mem\t# double" %}
 8285 
 8286   ins_encode( aarch64_enc_fldard(dst, mem) );
 8287 
 8288   ins_pipe(pipe_serial);
 8289 %}
 8290 
 8291 // Store Byte
 8292 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8293 %{
 8294   match(Set mem (StoreB mem src));
 8295 
 8296   ins_cost(VOLATILE_REF_COST);
 8297   format %{ "stlrb  $src, $mem\t# byte" %}
 8298 
 8299   ins_encode(aarch64_enc_stlrb(src, mem));
 8300 
 8301   ins_pipe(pipe_class_memory);
 8302 %}
 8303 
 8304 // Store Char/Short
 8305 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8306 %{
 8307   match(Set mem (StoreC mem src));
 8308 
 8309   ins_cost(VOLATILE_REF_COST);
 8310   format %{ "stlrh  $src, $mem\t# short" %}
 8311 
 8312   ins_encode(aarch64_enc_stlrh(src, mem));
 8313 
 8314   ins_pipe(pipe_class_memory);
 8315 %}
 8316 
 8317 // Store Integer
 8318 
 8319 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8320 %{
 8321   match(Set mem(StoreI mem src));
 8322 
 8323   ins_cost(VOLATILE_REF_COST);
 8324   format %{ "stlrw  $src, $mem\t# int" %}
 8325 
 8326   ins_encode(aarch64_enc_stlrw(src, mem));
 8327 
 8328   ins_pipe(pipe_class_memory);
 8329 %}
 8330 
 8331 // Store Long (64 bit signed)
 8332 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8333 %{
 8334   match(Set mem (StoreL mem src));
 8335 
 8336   ins_cost(VOLATILE_REF_COST);
 8337   format %{ "stlr  $src, $mem\t# int" %}
 8338 
 8339   ins_encode(aarch64_enc_stlr(src, mem));
 8340 
 8341   ins_pipe(pipe_class_memory);
 8342 %}
 8343 
 8344 // Store Pointer
 8345 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8346 %{
 8347   match(Set mem (StoreP mem src));
 8348 
 8349   ins_cost(VOLATILE_REF_COST);
 8350   format %{ "stlr  $src, $mem\t# ptr" %}
 8351 
 8352   ins_encode(aarch64_enc_stlr(src, mem));
 8353 
 8354   ins_pipe(pipe_class_memory);
 8355 %}
 8356 
 8357 // Store Compressed Pointer
 8358 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8359 %{
 8360   match(Set mem (StoreN mem src));
 8361 
 8362   ins_cost(VOLATILE_REF_COST);
 8363   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8364 
 8365   ins_encode(aarch64_enc_stlrw(src, mem));
 8366 
 8367   ins_pipe(pipe_class_memory);
 8368 %}
 8369 
 8370 // Store Float
 8371 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8372 %{
 8373   match(Set mem (StoreF mem src));
 8374 
 8375   ins_cost(VOLATILE_REF_COST);
 8376   format %{ "stlrs  $src, $mem\t# float" %}
 8377 
 8378   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8379 
 8380   ins_pipe(pipe_class_memory);
 8381 %}
 8382 
 8383 // TODO
 8384 // implement storeImmF0 and storeFImmPacked
 8385 
 8386 // Store Double
 8387 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8388 %{
 8389   match(Set mem (StoreD mem src));
 8390 
 8391   ins_cost(VOLATILE_REF_COST);
 8392   format %{ "stlrd  $src, $mem\t# double" %}
 8393 
 8394   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8395 
 8396   ins_pipe(pipe_class_memory);
 8397 %}
 8398 
 8399 //  ---------------- end of volatile loads and stores ----------------
 8400 
 8401 instruct cacheWB(indirect addr)
 8402 %{
 8403   predicate(VM_Version::supports_data_cache_line_flush());
 8404   match(CacheWB addr);
 8405 
 8406   ins_cost(100);
 8407   format %{"cache wb $addr" %}
 8408   ins_encode %{
 8409     assert($addr->index_position() < 0, "should be");
 8410     assert($addr$$disp == 0, "should be");
 8411     __ cache_wb(Address($addr$$base$$Register, 0));
 8412   %}
 8413   ins_pipe(pipe_slow); // XXX
 8414 %}
 8415 
 8416 instruct cacheWBPreSync()
 8417 %{
 8418   predicate(VM_Version::supports_data_cache_line_flush());
 8419   match(CacheWBPreSync);
 8420 
 8421   ins_cost(100);
 8422   format %{"cache wb presync" %}
 8423   ins_encode %{
 8424     __ cache_wbsync(true);
 8425   %}
 8426   ins_pipe(pipe_slow); // XXX
 8427 %}
 8428 
 8429 instruct cacheWBPostSync()
 8430 %{
 8431   predicate(VM_Version::supports_data_cache_line_flush());
 8432   match(CacheWBPostSync);
 8433 
 8434   ins_cost(100);
 8435   format %{"cache wb postsync" %}
 8436   ins_encode %{
 8437     __ cache_wbsync(false);
 8438   %}
 8439   ins_pipe(pipe_slow); // XXX
 8440 %}
 8441 
 8442 // ============================================================================
 8443 // BSWAP Instructions
 8444 
 8445 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8446   match(Set dst (ReverseBytesI src));
 8447 
 8448   ins_cost(INSN_COST);
 8449   format %{ "revw  $dst, $src" %}
 8450 
 8451   ins_encode %{
 8452     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8453   %}
 8454 
 8455   ins_pipe(ialu_reg);
 8456 %}
 8457 
 8458 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8459   match(Set dst (ReverseBytesL src));
 8460 
 8461   ins_cost(INSN_COST);
 8462   format %{ "rev  $dst, $src" %}
 8463 
 8464   ins_encode %{
 8465     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8466   %}
 8467 
 8468   ins_pipe(ialu_reg);
 8469 %}
 8470 
 8471 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8472   match(Set dst (ReverseBytesUS src));
 8473 
 8474   ins_cost(INSN_COST);
 8475   format %{ "rev16w  $dst, $src" %}
 8476 
 8477   ins_encode %{
 8478     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8479   %}
 8480 
 8481   ins_pipe(ialu_reg);
 8482 %}
 8483 
 8484 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8485   match(Set dst (ReverseBytesS src));
 8486 
 8487   ins_cost(INSN_COST);
 8488   format %{ "rev16w  $dst, $src\n\t"
 8489             "sbfmw $dst, $dst, #0, #15" %}
 8490 
 8491   ins_encode %{
 8492     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8493     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8494   %}
 8495 
 8496   ins_pipe(ialu_reg);
 8497 %}
 8498 
 8499 // ============================================================================
 8500 // Zero Count Instructions
 8501 
 8502 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8503   match(Set dst (CountLeadingZerosI src));
 8504 
 8505   ins_cost(INSN_COST);
 8506   format %{ "clzw  $dst, $src" %}
 8507   ins_encode %{
 8508     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8509   %}
 8510 
 8511   ins_pipe(ialu_reg);
 8512 %}
 8513 
 8514 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8515   match(Set dst (CountLeadingZerosL src));
 8516 
 8517   ins_cost(INSN_COST);
 8518   format %{ "clz   $dst, $src" %}
 8519   ins_encode %{
 8520     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8521   %}
 8522 
 8523   ins_pipe(ialu_reg);
 8524 %}
 8525 
 8526 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8527   match(Set dst (CountTrailingZerosI src));
 8528 
 8529   ins_cost(INSN_COST * 2);
 8530   format %{ "rbitw  $dst, $src\n\t"
 8531             "clzw   $dst, $dst" %}
 8532   ins_encode %{
 8533     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8534     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8535   %}
 8536 
 8537   ins_pipe(ialu_reg);
 8538 %}
 8539 
 8540 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8541   match(Set dst (CountTrailingZerosL src));
 8542 
 8543   ins_cost(INSN_COST * 2);
 8544   format %{ "rbit   $dst, $src\n\t"
 8545             "clz    $dst, $dst" %}
 8546   ins_encode %{
 8547     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8548     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8549   %}
 8550 
 8551   ins_pipe(ialu_reg);
 8552 %}
 8553 
 8554 //---------- Population Count Instructions -------------------------------------
 8555 //
 8556 
 8557 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8558   predicate(UsePopCountInstruction);
 8559   match(Set dst (PopCountI src));
 8560   effect(TEMP tmp);
 8561   ins_cost(INSN_COST * 13);
 8562 
 8563   format %{ "movw   $src, $src\n\t"
 8564             "mov    $tmp, $src\t# vector (1D)\n\t"
 8565             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8566             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8567             "mov    $dst, $tmp\t# vector (1D)" %}
 8568   ins_encode %{
 8569     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8570     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8571     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8572     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8573     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8574   %}
 8575 
 8576   ins_pipe(pipe_class_default);
 8577 %}
 8578 
 8579 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8580   predicate(UsePopCountInstruction);
 8581   match(Set dst (PopCountI (LoadI mem)));
 8582   effect(TEMP tmp);
 8583   ins_cost(INSN_COST * 13);
 8584 
 8585   format %{ "ldrs   $tmp, $mem\n\t"
 8586             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8587             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8588             "mov    $dst, $tmp\t# vector (1D)" %}
 8589   ins_encode %{
 8590     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8591     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8592               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8593     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8594     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8595     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8596   %}
 8597 
 8598   ins_pipe(pipe_class_default);
 8599 %}
 8600 
 8601 // Note: Long.bitCount(long) returns an int.
 8602 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8603   predicate(UsePopCountInstruction);
 8604   match(Set dst (PopCountL src));
 8605   effect(TEMP tmp);
 8606   ins_cost(INSN_COST * 13);
 8607 
 8608   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8609             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8610             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8611             "mov    $dst, $tmp\t# vector (1D)" %}
 8612   ins_encode %{
 8613     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8614     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8615     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8616     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8617   %}
 8618 
 8619   ins_pipe(pipe_class_default);
 8620 %}
 8621 
 8622 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8623   predicate(UsePopCountInstruction);
 8624   match(Set dst (PopCountL (LoadL mem)));
 8625   effect(TEMP tmp);
 8626   ins_cost(INSN_COST * 13);
 8627 
 8628   format %{ "ldrd   $tmp, $mem\n\t"
 8629             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8630             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8631             "mov    $dst, $tmp\t# vector (1D)" %}
 8632   ins_encode %{
 8633     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8634     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8635               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8636     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8637     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8638     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8639   %}
 8640 
 8641   ins_pipe(pipe_class_default);
 8642 %}
 8643 
 8644 // ============================================================================
 8645 // MemBar Instruction
 8646 
 8647 instruct load_fence() %{
 8648   match(LoadFence);
 8649   ins_cost(VOLATILE_REF_COST);
 8650 
 8651   format %{ "load_fence" %}
 8652 
 8653   ins_encode %{
 8654     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8655   %}
 8656   ins_pipe(pipe_serial);
 8657 %}
 8658 
 8659 instruct unnecessary_membar_acquire() %{
 8660   predicate(unnecessary_acquire(n));
 8661   match(MemBarAcquire);
 8662   ins_cost(0);
 8663 
 8664   format %{ "membar_acquire (elided)" %}
 8665 
 8666   ins_encode %{
 8667     __ block_comment("membar_acquire (elided)");
 8668   %}
 8669 
 8670   ins_pipe(pipe_class_empty);
 8671 %}
 8672 
 8673 instruct membar_acquire() %{
 8674   match(MemBarAcquire);
 8675   ins_cost(VOLATILE_REF_COST);
 8676 
 8677   format %{ "membar_acquire\n\t"
 8678             "dmb ish" %}
 8679 
 8680   ins_encode %{
 8681     __ block_comment("membar_acquire");
 8682     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8683   %}
 8684 
 8685   ins_pipe(pipe_serial);
 8686 %}
 8687 
 8688 
 8689 instruct membar_acquire_lock() %{
 8690   match(MemBarAcquireLock);
 8691   ins_cost(VOLATILE_REF_COST);
 8692 
 8693   format %{ "membar_acquire_lock (elided)" %}
 8694 
 8695   ins_encode %{
 8696     __ block_comment("membar_acquire_lock (elided)");
 8697   %}
 8698 
 8699   ins_pipe(pipe_serial);
 8700 %}
 8701 
 8702 instruct store_fence() %{
 8703   match(StoreFence);
 8704   ins_cost(VOLATILE_REF_COST);
 8705 
 8706   format %{ "store_fence" %}
 8707 
 8708   ins_encode %{
 8709     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8710   %}
 8711   ins_pipe(pipe_serial);
 8712 %}
 8713 
 8714 instruct unnecessary_membar_release() %{
 8715   predicate(unnecessary_release(n));
 8716   match(MemBarRelease);
 8717   ins_cost(0);
 8718 
 8719   format %{ "membar_release (elided)" %}
 8720 
 8721   ins_encode %{
 8722     __ block_comment("membar_release (elided)");
 8723   %}
 8724   ins_pipe(pipe_serial);
 8725 %}
 8726 
 8727 instruct membar_release() %{
 8728   match(MemBarRelease);
 8729   ins_cost(VOLATILE_REF_COST);
 8730 
 8731   format %{ "membar_release\n\t"
 8732             "dmb ish" %}
 8733 
 8734   ins_encode %{
 8735     __ block_comment("membar_release");
 8736     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8737   %}
 8738   ins_pipe(pipe_serial);
 8739 %}
 8740 
 8741 instruct membar_storestore() %{
 8742   match(MemBarStoreStore);
 8743   match(StoreStoreFence);
 8744   ins_cost(VOLATILE_REF_COST);
 8745 
 8746   format %{ "MEMBAR-store-store" %}
 8747 
 8748   ins_encode %{
 8749     __ membar(Assembler::StoreStore);
 8750   %}
 8751   ins_pipe(pipe_serial);
 8752 %}
 8753 
 8754 instruct membar_release_lock() %{
 8755   match(MemBarReleaseLock);
 8756   ins_cost(VOLATILE_REF_COST);
 8757 
 8758   format %{ "membar_release_lock (elided)" %}
 8759 
 8760   ins_encode %{
 8761     __ block_comment("membar_release_lock (elided)");
 8762   %}
 8763 
 8764   ins_pipe(pipe_serial);
 8765 %}
 8766 
 8767 instruct unnecessary_membar_volatile() %{
 8768   predicate(unnecessary_volatile(n));
 8769   match(MemBarVolatile);
 8770   ins_cost(0);
 8771 
 8772   format %{ "membar_volatile (elided)" %}
 8773 
 8774   ins_encode %{
 8775     __ block_comment("membar_volatile (elided)");
 8776   %}
 8777 
 8778   ins_pipe(pipe_serial);
 8779 %}
 8780 
 8781 instruct membar_volatile() %{
 8782   match(MemBarVolatile);
 8783   ins_cost(VOLATILE_REF_COST*100);
 8784 
 8785   format %{ "membar_volatile\n\t"
 8786              "dmb ish"%}
 8787 
 8788   ins_encode %{
 8789     __ block_comment("membar_volatile");
 8790     __ membar(Assembler::StoreLoad);
 8791   %}
 8792 
 8793   ins_pipe(pipe_serial);
 8794 %}
 8795 
 8796 // ============================================================================
 8797 // Cast/Convert Instructions
 8798 
 8799 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8800   match(Set dst (CastX2P src));
 8801 
 8802   ins_cost(INSN_COST);
 8803   format %{ "mov $dst, $src\t# long -> ptr" %}
 8804 
 8805   ins_encode %{
 8806     if ($dst$$reg != $src$$reg) {
 8807       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8808     }
 8809   %}
 8810 
 8811   ins_pipe(ialu_reg);
 8812 %}
 8813 
 8814 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8815   match(Set dst (CastP2X src));
 8816 
 8817   ins_cost(INSN_COST);
 8818   format %{ "mov $dst, $src\t# ptr -> long" %}
 8819 
 8820   ins_encode %{
 8821     if ($dst$$reg != $src$$reg) {
 8822       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8823     }
 8824   %}
 8825 
 8826   ins_pipe(ialu_reg);
 8827 %}
 8828 
 8829 // Convert oop into int for vectors alignment masking
 8830 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8831   match(Set dst (ConvL2I (CastP2X src)));
 8832 
 8833   ins_cost(INSN_COST);
 8834   format %{ "movw $dst, $src\t# ptr -> int" %}
 8835   ins_encode %{
 8836     __ movw($dst$$Register, $src$$Register);
 8837   %}
 8838 
 8839   ins_pipe(ialu_reg);
 8840 %}
 8841 
 8842 // Convert compressed oop into int for vectors alignment masking
 8843 // in case of 32bit oops (heap < 4Gb).
 8844 instruct convN2I(iRegINoSp dst, iRegN src)
 8845 %{
 8846   predicate(CompressedOops::shift() == 0);
 8847   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8848 
 8849   ins_cost(INSN_COST);
 8850   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8851   ins_encode %{
 8852     __ movw($dst$$Register, $src$$Register);
 8853   %}
 8854 
 8855   ins_pipe(ialu_reg);
 8856 %}
 8857 
 8858 
 8859 // Convert oop pointer into compressed form
 8860 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8861   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8862   match(Set dst (EncodeP src));
 8863   effect(KILL cr);
 8864   ins_cost(INSN_COST * 3);
 8865   format %{ "encode_heap_oop $dst, $src" %}
 8866   ins_encode %{
 8867     Register s = $src$$Register;
 8868     Register d = $dst$$Register;
 8869     __ encode_heap_oop(d, s);
 8870   %}
 8871   ins_pipe(ialu_reg);
 8872 %}
 8873 
 8874 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8875   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8876   match(Set dst (EncodeP src));
 8877   ins_cost(INSN_COST * 3);
 8878   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8879   ins_encode %{
 8880     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8881   %}
 8882   ins_pipe(ialu_reg);
 8883 %}
 8884 
 8885 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8886   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8887             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8888   match(Set dst (DecodeN src));
 8889   ins_cost(INSN_COST * 3);
 8890   format %{ "decode_heap_oop $dst, $src" %}
 8891   ins_encode %{
 8892     Register s = $src$$Register;
 8893     Register d = $dst$$Register;
 8894     __ decode_heap_oop(d, s);
 8895   %}
 8896   ins_pipe(ialu_reg);
 8897 %}
 8898 
 8899 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8900   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8901             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8902   match(Set dst (DecodeN src));
 8903   ins_cost(INSN_COST * 3);
 8904   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8905   ins_encode %{
 8906     Register s = $src$$Register;
 8907     Register d = $dst$$Register;
 8908     __ decode_heap_oop_not_null(d, s);
 8909   %}
 8910   ins_pipe(ialu_reg);
 8911 %}
 8912 
 8913 // n.b. AArch64 implementations of encode_klass_not_null and
 8914 // decode_klass_not_null do not modify the flags register so, unlike
 8915 // Intel, we don't kill CR as a side effect here
 8916 
 8917 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8918   match(Set dst (EncodePKlass src));
 8919 
 8920   ins_cost(INSN_COST * 3);
 8921   format %{ "encode_klass_not_null $dst,$src" %}
 8922 
 8923   ins_encode %{
 8924     Register src_reg = as_Register($src$$reg);
 8925     Register dst_reg = as_Register($dst$$reg);
 8926     __ encode_klass_not_null(dst_reg, src_reg);
 8927   %}
 8928 
 8929    ins_pipe(ialu_reg);
 8930 %}
 8931 
 8932 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8933   match(Set dst (DecodeNKlass src));
 8934 
 8935   ins_cost(INSN_COST * 3);
 8936   format %{ "decode_klass_not_null $dst,$src" %}
 8937 
 8938   ins_encode %{
 8939     Register src_reg = as_Register($src$$reg);
 8940     Register dst_reg = as_Register($dst$$reg);
 8941     if (dst_reg != src_reg) {
 8942       __ decode_klass_not_null(dst_reg, src_reg);
 8943     } else {
 8944       __ decode_klass_not_null(dst_reg);
 8945     }
 8946   %}
 8947 
 8948    ins_pipe(ialu_reg);
 8949 %}
 8950 
 8951 instruct checkCastPP(iRegPNoSp dst)
 8952 %{
 8953   match(Set dst (CheckCastPP dst));
 8954 
 8955   size(0);
 8956   format %{ "# checkcastPP of $dst" %}
 8957   ins_encode(/* empty encoding */);
 8958   ins_pipe(pipe_class_empty);
 8959 %}
 8960 
 8961 instruct castPP(iRegPNoSp dst)
 8962 %{
 8963   match(Set dst (CastPP dst));
 8964 
 8965   size(0);
 8966   format %{ "# castPP of $dst" %}
 8967   ins_encode(/* empty encoding */);
 8968   ins_pipe(pipe_class_empty);
 8969 %}
 8970 
 8971 instruct castII(iRegI dst)
 8972 %{
 8973   match(Set dst (CastII dst));
 8974 
 8975   size(0);
 8976   format %{ "# castII of $dst" %}
 8977   ins_encode(/* empty encoding */);
 8978   ins_cost(0);
 8979   ins_pipe(pipe_class_empty);
 8980 %}
 8981 
 8982 instruct castLL(iRegL dst)
 8983 %{
 8984   match(Set dst (CastLL dst));
 8985 
 8986   size(0);
 8987   format %{ "# castLL of $dst" %}
 8988   ins_encode(/* empty encoding */);
 8989   ins_cost(0);
 8990   ins_pipe(pipe_class_empty);
 8991 %}
 8992 
 8993 instruct castFF(vRegF dst)
 8994 %{
 8995   match(Set dst (CastFF dst));
 8996 
 8997   size(0);
 8998   format %{ "# castFF of $dst" %}
 8999   ins_encode(/* empty encoding */);
 9000   ins_cost(0);
 9001   ins_pipe(pipe_class_empty);
 9002 %}
 9003 
 9004 instruct castDD(vRegD dst)
 9005 %{
 9006   match(Set dst (CastDD dst));
 9007 
 9008   size(0);
 9009   format %{ "# castDD of $dst" %}
 9010   ins_encode(/* empty encoding */);
 9011   ins_cost(0);
 9012   ins_pipe(pipe_class_empty);
 9013 %}
 9014 
 9015 instruct castVVD(vecD dst)
 9016 %{
 9017   match(Set dst (CastVV dst));
 9018 
 9019   size(0);
 9020   format %{ "# castVV of $dst" %}
 9021   ins_encode(/* empty encoding */);
 9022   ins_cost(0);
 9023   ins_pipe(pipe_class_empty);
 9024 %}
 9025 
 9026 instruct castVVX(vecX dst)
 9027 %{
 9028   match(Set dst (CastVV dst));
 9029 
 9030   size(0);
 9031   format %{ "# castVV of $dst" %}
 9032   ins_encode(/* empty encoding */);
 9033   ins_cost(0);
 9034   ins_pipe(pipe_class_empty);
 9035 %}
 9036 
 9037 instruct castVV(vReg dst)
 9038 %{
 9039   match(Set dst (CastVV dst));
 9040 
 9041   size(0);
 9042   format %{ "# castVV of $dst" %}
 9043   ins_encode(/* empty encoding */);
 9044   ins_cost(0);
 9045   ins_pipe(pipe_class_empty);
 9046 %}
 9047 
 9048 instruct castVVMask(pRegGov dst)
 9049 %{
 9050   match(Set dst (CastVV dst));
 9051 
 9052   size(0);
 9053   format %{ "# castVV of $dst" %}
 9054   ins_encode(/* empty encoding */);
 9055   ins_cost(0);
 9056   ins_pipe(pipe_class_empty);
 9057 %}
 9058 
 9059 // ============================================================================
 9060 // Atomic operation instructions
 9061 //
 9062 // Intel and SPARC both implement Ideal Node LoadPLocked and
 9063 // Store{PIL}Conditional instructions using a normal load for the
 9064 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 9065 //
 9066 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 9067 // pair to lock object allocations from Eden space when not using
 9068 // TLABs.
 9069 //
 9070 // There does not appear to be a Load{IL}Locked Ideal Node and the
 9071 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 9072 // and to use StoreIConditional only for 32-bit and StoreLConditional
 9073 // only for 64-bit.
 9074 //
 9075 // We implement LoadPLocked and StorePLocked instructions using,
 9076 // respectively the AArch64 hw load-exclusive and store-conditional
 9077 // instructions. Whereas we must implement each of
 9078 // Store{IL}Conditional using a CAS which employs a pair of
 9079 // instructions comprising a load-exclusive followed by a
 9080 // store-conditional.
 9081 
 9082 
 9083 // Locked-load (linked load) of the current heap-top
 9084 // used when updating the eden heap top
 9085 // implemented using ldaxr on AArch64
 9086 
 9087 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 9088 %{
 9089   match(Set dst (LoadPLocked mem));
 9090 
 9091   ins_cost(VOLATILE_REF_COST);
 9092 
 9093   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 9094 
 9095   ins_encode(aarch64_enc_ldaxr(dst, mem));
 9096 
 9097   ins_pipe(pipe_serial);
 9098 %}
 9099 
 9100 // Conditional-store of the updated heap-top.
 9101 // Used during allocation of the shared heap.
 9102 // Sets flag (EQ) on success.
 9103 // implemented using stlxr on AArch64.
 9104 
 9105 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9106 %{
 9107   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9108 
 9109   ins_cost(VOLATILE_REF_COST);
 9110 
 9111  // TODO
 9112  // do we need to do a store-conditional release or can we just use a
 9113  // plain store-conditional?
 9114 
 9115   format %{
 9116     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9117     "cmpw rscratch1, zr\t# EQ on successful write"
 9118   %}
 9119 
 9120   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9121 
 9122   ins_pipe(pipe_serial);
 9123 %}
 9124 
 9125 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9126 %{
 9127   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9128 
 9129   ins_cost(VOLATILE_REF_COST);
 9130 
 9131   format %{
 9132     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9133     "cmpw rscratch1, zr\t# EQ on successful write"
 9134   %}
 9135 
 9136   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9137 
 9138   ins_pipe(pipe_slow);
 9139 %}
 9140 
 9141 // storeIConditional also has acquire semantics, for no better reason
 9142 // than matching storeLConditional.  At the time of writing this
 9143 // comment storeIConditional was not used anywhere by AArch64.
 9144 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9145 %{
 9146   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9147 
 9148   ins_cost(VOLATILE_REF_COST);
 9149 
 9150   format %{
 9151     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9152     "cmpw rscratch1, zr\t# EQ on successful write"
 9153   %}
 9154 
 9155   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9156 
 9157   ins_pipe(pipe_slow);
 9158 %}
 9159 
 9160 // standard CompareAndSwapX when we are using barriers
 9161 // these have higher priority than the rules selected by a predicate
 9162 
 9163 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9164 // can't match them
 9165 
 9166 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9167 
 9168   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9169   ins_cost(2 * VOLATILE_REF_COST);
 9170 
 9171   effect(KILL cr);
 9172 
 9173   format %{
 9174     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9175     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9176   %}
 9177 
 9178   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9179             aarch64_enc_cset_eq(res));
 9180 
 9181   ins_pipe(pipe_slow);
 9182 %}
 9183 
 9184 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9185 
 9186   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9187   ins_cost(2 * VOLATILE_REF_COST);
 9188 
 9189   effect(KILL cr);
 9190 
 9191   format %{
 9192     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9193     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9194   %}
 9195 
 9196   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9197             aarch64_enc_cset_eq(res));
 9198 
 9199   ins_pipe(pipe_slow);
 9200 %}
 9201 
 9202 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9203 
 9204   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9205   ins_cost(2 * VOLATILE_REF_COST);
 9206 
 9207   effect(KILL cr);
 9208 
 9209  format %{
 9210     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9211     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9212  %}
 9213 
 9214  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9215             aarch64_enc_cset_eq(res));
 9216 
 9217   ins_pipe(pipe_slow);
 9218 %}
 9219 
 9220 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9221 
 9222   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9223   ins_cost(2 * VOLATILE_REF_COST);
 9224 
 9225   effect(KILL cr);
 9226 
 9227  format %{
 9228     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9229     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9230  %}
 9231 
 9232  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9233             aarch64_enc_cset_eq(res));
 9234 
 9235   ins_pipe(pipe_slow);
 9236 %}
 9237 
 9238 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9239 
 9240   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9241   predicate(n->as_LoadStore()->barrier_data() == 0);
 9242   ins_cost(2 * VOLATILE_REF_COST);
 9243 
 9244   effect(KILL cr);
 9245 
 9246  format %{
 9247     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9248     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9249  %}
 9250 
 9251  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9252             aarch64_enc_cset_eq(res));
 9253 
 9254   ins_pipe(pipe_slow);
 9255 %}
 9256 
 9257 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9258 
 9259   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9260   ins_cost(2 * VOLATILE_REF_COST);
 9261 
 9262   effect(KILL cr);
 9263 
 9264  format %{
 9265     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9266     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9267  %}
 9268 
 9269  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9270             aarch64_enc_cset_eq(res));
 9271 
 9272   ins_pipe(pipe_slow);
 9273 %}
 9274 
 9275 // alternative CompareAndSwapX when we are eliding barriers
 9276 
 9277 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9278 
 9279   predicate(needs_acquiring_load_exclusive(n));
 9280   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9281   ins_cost(VOLATILE_REF_COST);
 9282 
 9283   effect(KILL cr);
 9284 
 9285   format %{
 9286     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9287     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9288   %}
 9289 
 9290   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9291             aarch64_enc_cset_eq(res));
 9292 
 9293   ins_pipe(pipe_slow);
 9294 %}
 9295 
 9296 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9297 
 9298   predicate(needs_acquiring_load_exclusive(n));
 9299   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9300   ins_cost(VOLATILE_REF_COST);
 9301 
 9302   effect(KILL cr);
 9303 
 9304   format %{
 9305     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9306     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9307   %}
 9308 
 9309   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9310             aarch64_enc_cset_eq(res));
 9311 
 9312   ins_pipe(pipe_slow);
 9313 %}
 9314 
 9315 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9316 
 9317   predicate(needs_acquiring_load_exclusive(n));
 9318   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9319   ins_cost(VOLATILE_REF_COST);
 9320 
 9321   effect(KILL cr);
 9322 
 9323  format %{
 9324     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9325     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9326  %}
 9327 
 9328  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9329             aarch64_enc_cset_eq(res));
 9330 
 9331   ins_pipe(pipe_slow);
 9332 %}
 9333 
 9334 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9335 
 9336   predicate(needs_acquiring_load_exclusive(n));
 9337   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9338   ins_cost(VOLATILE_REF_COST);
 9339 
 9340   effect(KILL cr);
 9341 
 9342  format %{
 9343     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9344     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9345  %}
 9346 
 9347  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9348             aarch64_enc_cset_eq(res));
 9349 
 9350   ins_pipe(pipe_slow);
 9351 %}
 9352 
 9353 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9354 
 9355   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9356   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9357   ins_cost(VOLATILE_REF_COST);
 9358 
 9359   effect(KILL cr);
 9360 
 9361  format %{
 9362     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9363     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9364  %}
 9365 
 9366  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9367             aarch64_enc_cset_eq(res));
 9368 
 9369   ins_pipe(pipe_slow);
 9370 %}
 9371 
 9372 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9373 
 9374   predicate(needs_acquiring_load_exclusive(n));
 9375   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9376   ins_cost(VOLATILE_REF_COST);
 9377 
 9378   effect(KILL cr);
 9379 
 9380  format %{
 9381     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9382     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9383  %}
 9384 
 9385  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9386             aarch64_enc_cset_eq(res));
 9387 
 9388   ins_pipe(pipe_slow);
 9389 %}
 9390 
 9391 
 9392 // ---------------------------------------------------------------------
 9393 
 9394 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9395 
 9396 // Sundry CAS operations.  Note that release is always true,
 9397 // regardless of the memory ordering of the CAS.  This is because we
 9398 // need the volatile case to be sequentially consistent but there is
 9399 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9400 // can't check the type of memory ordering here, so we always emit a
 9401 // STLXR.
 9402 
 9403 // This section is generated from aarch64_ad_cas.m4
 9404 
 9405 
 9406 
 9407 // This pattern is generated automatically from cas.m4.
 9408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9409 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9410 
 9411   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9412   ins_cost(2 * VOLATILE_REF_COST);
 9413   effect(TEMP_DEF res, KILL cr);
 9414   format %{
 9415     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9416   %}
 9417   ins_encode %{
 9418     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9419                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9420                /*weak*/ false, $res$$Register);
 9421     __ sxtbw($res$$Register, $res$$Register);
 9422   %}
 9423   ins_pipe(pipe_slow);
 9424 %}
 9425 
 9426 // This pattern is generated automatically from cas.m4.
 9427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9428 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9429 
 9430   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9431   ins_cost(2 * VOLATILE_REF_COST);
 9432   effect(TEMP_DEF res, KILL cr);
 9433   format %{
 9434     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9435   %}
 9436   ins_encode %{
 9437     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9438                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9439                /*weak*/ false, $res$$Register);
 9440     __ sxthw($res$$Register, $res$$Register);
 9441   %}
 9442   ins_pipe(pipe_slow);
 9443 %}
 9444 
 9445 // This pattern is generated automatically from cas.m4.
 9446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9447 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9448 
 9449   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9450   ins_cost(2 * VOLATILE_REF_COST);
 9451   effect(TEMP_DEF res, KILL cr);
 9452   format %{
 9453     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9454   %}
 9455   ins_encode %{
 9456     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9457                Assembler::word, /*acquire*/ false, /*release*/ true,
 9458                /*weak*/ false, $res$$Register);
 9459   %}
 9460   ins_pipe(pipe_slow);
 9461 %}
 9462 
 9463 // This pattern is generated automatically from cas.m4.
 9464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9465 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9466 
 9467   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9468   ins_cost(2 * VOLATILE_REF_COST);
 9469   effect(TEMP_DEF res, KILL cr);
 9470   format %{
 9471     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9472   %}
 9473   ins_encode %{
 9474     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9475                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9476                /*weak*/ false, $res$$Register);
 9477   %}
 9478   ins_pipe(pipe_slow);
 9479 %}
 9480 
 9481 // This pattern is generated automatically from cas.m4.
 9482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9483 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9484 
 9485   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9486   ins_cost(2 * VOLATILE_REF_COST);
 9487   effect(TEMP_DEF res, KILL cr);
 9488   format %{
 9489     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9490   %}
 9491   ins_encode %{
 9492     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9493                Assembler::word, /*acquire*/ false, /*release*/ true,
 9494                /*weak*/ false, $res$$Register);
 9495   %}
 9496   ins_pipe(pipe_slow);
 9497 %}
 9498 
 9499 // This pattern is generated automatically from cas.m4.
 9500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9501 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9502   predicate(n->as_LoadStore()->barrier_data() == 0);
 9503   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9504   ins_cost(2 * VOLATILE_REF_COST);
 9505   effect(TEMP_DEF res, KILL cr);
 9506   format %{
 9507     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9508   %}
 9509   ins_encode %{
 9510     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9511                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9512                /*weak*/ false, $res$$Register);
 9513   %}
 9514   ins_pipe(pipe_slow);
 9515 %}
 9516 
 9517 // This pattern is generated automatically from cas.m4.
 9518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9519 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9520   predicate(needs_acquiring_load_exclusive(n));
 9521   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9522   ins_cost(VOLATILE_REF_COST);
 9523   effect(TEMP_DEF res, KILL cr);
 9524   format %{
 9525     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9526   %}
 9527   ins_encode %{
 9528     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9529                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9530                /*weak*/ false, $res$$Register);
 9531     __ sxtbw($res$$Register, $res$$Register);
 9532   %}
 9533   ins_pipe(pipe_slow);
 9534 %}
 9535 
 9536 // This pattern is generated automatically from cas.m4.
 9537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9538 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9539   predicate(needs_acquiring_load_exclusive(n));
 9540   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9541   ins_cost(VOLATILE_REF_COST);
 9542   effect(TEMP_DEF res, KILL cr);
 9543   format %{
 9544     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9545   %}
 9546   ins_encode %{
 9547     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9548                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9549                /*weak*/ false, $res$$Register);
 9550     __ sxthw($res$$Register, $res$$Register);
 9551   %}
 9552   ins_pipe(pipe_slow);
 9553 %}
 9554 
 9555 // This pattern is generated automatically from cas.m4.
 9556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9557 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9558   predicate(needs_acquiring_load_exclusive(n));
 9559   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9560   ins_cost(VOLATILE_REF_COST);
 9561   effect(TEMP_DEF res, KILL cr);
 9562   format %{
 9563     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9564   %}
 9565   ins_encode %{
 9566     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9567                Assembler::word, /*acquire*/ true, /*release*/ true,
 9568                /*weak*/ false, $res$$Register);
 9569   %}
 9570   ins_pipe(pipe_slow);
 9571 %}
 9572 
 9573 // This pattern is generated automatically from cas.m4.
 9574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9575 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9576   predicate(needs_acquiring_load_exclusive(n));
 9577   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9578   ins_cost(VOLATILE_REF_COST);
 9579   effect(TEMP_DEF res, KILL cr);
 9580   format %{
 9581     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9582   %}
 9583   ins_encode %{
 9584     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9585                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9586                /*weak*/ false, $res$$Register);
 9587   %}
 9588   ins_pipe(pipe_slow);
 9589 %}
 9590 
 9591 // This pattern is generated automatically from cas.m4.
 9592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9593 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9594   predicate(needs_acquiring_load_exclusive(n));
 9595   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9596   ins_cost(VOLATILE_REF_COST);
 9597   effect(TEMP_DEF res, KILL cr);
 9598   format %{
 9599     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9600   %}
 9601   ins_encode %{
 9602     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9603                Assembler::word, /*acquire*/ true, /*release*/ true,
 9604                /*weak*/ false, $res$$Register);
 9605   %}
 9606   ins_pipe(pipe_slow);
 9607 %}
 9608 
 9609 // This pattern is generated automatically from cas.m4.
 9610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9611 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9612   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9613   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9614   ins_cost(VOLATILE_REF_COST);
 9615   effect(TEMP_DEF res, KILL cr);
 9616   format %{
 9617     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9618   %}
 9619   ins_encode %{
 9620     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9621                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9622                /*weak*/ false, $res$$Register);
 9623   %}
 9624   ins_pipe(pipe_slow);
 9625 %}
 9626 
 9627 // This pattern is generated automatically from cas.m4.
 9628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9629 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9630 
 9631   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9632   ins_cost(2 * VOLATILE_REF_COST);
 9633   effect(KILL cr);
 9634   format %{
 9635     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9636     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9637   %}
 9638   ins_encode %{
 9639     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9640                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9641                /*weak*/ true, noreg);
 9642     __ csetw($res$$Register, Assembler::EQ);
 9643   %}
 9644   ins_pipe(pipe_slow);
 9645 %}
 9646 
 9647 // This pattern is generated automatically from cas.m4.
 9648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9649 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9650 
 9651   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9652   ins_cost(2 * VOLATILE_REF_COST);
 9653   effect(KILL cr);
 9654   format %{
 9655     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9656     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9657   %}
 9658   ins_encode %{
 9659     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9660                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9661                /*weak*/ true, noreg);
 9662     __ csetw($res$$Register, Assembler::EQ);
 9663   %}
 9664   ins_pipe(pipe_slow);
 9665 %}
 9666 
 9667 // This pattern is generated automatically from cas.m4.
 9668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9669 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9670 
 9671   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9672   ins_cost(2 * VOLATILE_REF_COST);
 9673   effect(KILL cr);
 9674   format %{
 9675     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9676     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9677   %}
 9678   ins_encode %{
 9679     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9680                Assembler::word, /*acquire*/ false, /*release*/ true,
 9681                /*weak*/ true, noreg);
 9682     __ csetw($res$$Register, Assembler::EQ);
 9683   %}
 9684   ins_pipe(pipe_slow);
 9685 %}
 9686 
 9687 // This pattern is generated automatically from cas.m4.
 9688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9689 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9690 
 9691   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9692   ins_cost(2 * VOLATILE_REF_COST);
 9693   effect(KILL cr);
 9694   format %{
 9695     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9696     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9697   %}
 9698   ins_encode %{
 9699     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9700                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9701                /*weak*/ true, noreg);
 9702     __ csetw($res$$Register, Assembler::EQ);
 9703   %}
 9704   ins_pipe(pipe_slow);
 9705 %}
 9706 
 9707 // This pattern is generated automatically from cas.m4.
 9708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9709 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9710 
 9711   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9712   ins_cost(2 * VOLATILE_REF_COST);
 9713   effect(KILL cr);
 9714   format %{
 9715     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9716     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9717   %}
 9718   ins_encode %{
 9719     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9720                Assembler::word, /*acquire*/ false, /*release*/ true,
 9721                /*weak*/ true, noreg);
 9722     __ csetw($res$$Register, Assembler::EQ);
 9723   %}
 9724   ins_pipe(pipe_slow);
 9725 %}
 9726 
 9727 // This pattern is generated automatically from cas.m4.
 9728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9729 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9730   predicate(n->as_LoadStore()->barrier_data() == 0);
 9731   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9732   ins_cost(2 * VOLATILE_REF_COST);
 9733   effect(KILL cr);
 9734   format %{
 9735     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9736     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9737   %}
 9738   ins_encode %{
 9739     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9740                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9741                /*weak*/ true, noreg);
 9742     __ csetw($res$$Register, Assembler::EQ);
 9743   %}
 9744   ins_pipe(pipe_slow);
 9745 %}
 9746 
 9747 // This pattern is generated automatically from cas.m4.
 9748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9749 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9750   predicate(needs_acquiring_load_exclusive(n));
 9751   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9752   ins_cost(VOLATILE_REF_COST);
 9753   effect(KILL cr);
 9754   format %{
 9755     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9756     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9757   %}
 9758   ins_encode %{
 9759     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9760                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9761                /*weak*/ true, noreg);
 9762     __ csetw($res$$Register, Assembler::EQ);
 9763   %}
 9764   ins_pipe(pipe_slow);
 9765 %}
 9766 
 9767 // This pattern is generated automatically from cas.m4.
 9768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9769 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9770   predicate(needs_acquiring_load_exclusive(n));
 9771   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9772   ins_cost(VOLATILE_REF_COST);
 9773   effect(KILL cr);
 9774   format %{
 9775     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9776     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9777   %}
 9778   ins_encode %{
 9779     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9780                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9781                /*weak*/ true, noreg);
 9782     __ csetw($res$$Register, Assembler::EQ);
 9783   %}
 9784   ins_pipe(pipe_slow);
 9785 %}
 9786 
 9787 // This pattern is generated automatically from cas.m4.
 9788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9789 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9790   predicate(needs_acquiring_load_exclusive(n));
 9791   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9792   ins_cost(VOLATILE_REF_COST);
 9793   effect(KILL cr);
 9794   format %{
 9795     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9796     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9797   %}
 9798   ins_encode %{
 9799     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9800                Assembler::word, /*acquire*/ true, /*release*/ true,
 9801                /*weak*/ true, noreg);
 9802     __ csetw($res$$Register, Assembler::EQ);
 9803   %}
 9804   ins_pipe(pipe_slow);
 9805 %}
 9806 
 9807 // This pattern is generated automatically from cas.m4.
 9808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9809 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9810   predicate(needs_acquiring_load_exclusive(n));
 9811   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9812   ins_cost(VOLATILE_REF_COST);
 9813   effect(KILL cr);
 9814   format %{
 9815     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9816     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9817   %}
 9818   ins_encode %{
 9819     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9820                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9821                /*weak*/ true, noreg);
 9822     __ csetw($res$$Register, Assembler::EQ);
 9823   %}
 9824   ins_pipe(pipe_slow);
 9825 %}
 9826 
 9827 // This pattern is generated automatically from cas.m4.
 9828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9829 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9830   predicate(needs_acquiring_load_exclusive(n));
 9831   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9832   ins_cost(VOLATILE_REF_COST);
 9833   effect(KILL cr);
 9834   format %{
 9835     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9836     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9837   %}
 9838   ins_encode %{
 9839     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9840                Assembler::word, /*acquire*/ true, /*release*/ true,
 9841                /*weak*/ true, noreg);
 9842     __ csetw($res$$Register, Assembler::EQ);
 9843   %}
 9844   ins_pipe(pipe_slow);
 9845 %}
 9846 
 9847 // This pattern is generated automatically from cas.m4.
 9848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9849 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9850   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9851   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9852   ins_cost(VOLATILE_REF_COST);
 9853   effect(KILL cr);
 9854   format %{
 9855     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9856     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9857   %}
 9858   ins_encode %{
 9859     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9860                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9861                /*weak*/ true, noreg);
 9862     __ csetw($res$$Register, Assembler::EQ);
 9863   %}
 9864   ins_pipe(pipe_slow);
 9865 %}
 9866 
 9867 // END This section of the file is automatically generated. Do not edit --------------
 9868 // ---------------------------------------------------------------------
 9869 
 9870 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9871   match(Set prev (GetAndSetI mem newv));
 9872   ins_cost(2 * VOLATILE_REF_COST);
 9873   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9874   ins_encode %{
 9875     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9876   %}
 9877   ins_pipe(pipe_serial);
 9878 %}
 9879 
 9880 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9881   match(Set prev (GetAndSetL mem newv));
 9882   ins_cost(2 * VOLATILE_REF_COST);
 9883   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9884   ins_encode %{
 9885     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9886   %}
 9887   ins_pipe(pipe_serial);
 9888 %}
 9889 
 9890 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9891   match(Set prev (GetAndSetN mem newv));
 9892   ins_cost(2 * VOLATILE_REF_COST);
 9893   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9894   ins_encode %{
 9895     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9896   %}
 9897   ins_pipe(pipe_serial);
 9898 %}
 9899 
 9900 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9901   predicate(n->as_LoadStore()->barrier_data() == 0);
 9902   match(Set prev (GetAndSetP mem newv));
 9903   ins_cost(2 * VOLATILE_REF_COST);
 9904   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9905   ins_encode %{
 9906     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9907   %}
 9908   ins_pipe(pipe_serial);
 9909 %}
 9910 
 9911 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9912   predicate(needs_acquiring_load_exclusive(n));
 9913   match(Set prev (GetAndSetI mem newv));
 9914   ins_cost(VOLATILE_REF_COST);
 9915   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9916   ins_encode %{
 9917     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9918   %}
 9919   ins_pipe(pipe_serial);
 9920 %}
 9921 
 9922 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9923   predicate(needs_acquiring_load_exclusive(n));
 9924   match(Set prev (GetAndSetL mem newv));
 9925   ins_cost(VOLATILE_REF_COST);
 9926   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9927   ins_encode %{
 9928     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9929   %}
 9930   ins_pipe(pipe_serial);
 9931 %}
 9932 
 9933 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9934   predicate(needs_acquiring_load_exclusive(n));
 9935   match(Set prev (GetAndSetN mem newv));
 9936   ins_cost(VOLATILE_REF_COST);
 9937   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9938   ins_encode %{
 9939     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9940   %}
 9941   ins_pipe(pipe_serial);
 9942 %}
 9943 
 9944 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9945   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9946   match(Set prev (GetAndSetP mem newv));
 9947   ins_cost(VOLATILE_REF_COST);
 9948   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9949   ins_encode %{
 9950     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9951   %}
 9952   ins_pipe(pipe_serial);
 9953 %}
 9954 
 9955 
 9956 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9957   match(Set newval (GetAndAddL mem incr));
 9958   ins_cost(2 * VOLATILE_REF_COST + 1);
 9959   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9960   ins_encode %{
 9961     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9962   %}
 9963   ins_pipe(pipe_serial);
 9964 %}
 9965 
 9966 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9967   predicate(n->as_LoadStore()->result_not_used());
 9968   match(Set dummy (GetAndAddL mem incr));
 9969   ins_cost(2 * VOLATILE_REF_COST);
 9970   format %{ "get_and_addL [$mem], $incr" %}
 9971   ins_encode %{
 9972     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9973   %}
 9974   ins_pipe(pipe_serial);
 9975 %}
 9976 
 9977 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9978   match(Set newval (GetAndAddL mem incr));
 9979   ins_cost(2 * VOLATILE_REF_COST + 1);
 9980   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9981   ins_encode %{
 9982     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9983   %}
 9984   ins_pipe(pipe_serial);
 9985 %}
 9986 
 9987 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9988   predicate(n->as_LoadStore()->result_not_used());
 9989   match(Set dummy (GetAndAddL mem incr));
 9990   ins_cost(2 * VOLATILE_REF_COST);
 9991   format %{ "get_and_addL [$mem], $incr" %}
 9992   ins_encode %{
 9993     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9994   %}
 9995   ins_pipe(pipe_serial);
 9996 %}
 9997 
 9998 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9999   match(Set newval (GetAndAddI mem incr));
10000   ins_cost(2 * VOLATILE_REF_COST + 1);
10001   format %{ "get_and_addI $newval, [$mem], $incr" %}
10002   ins_encode %{
10003     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10004   %}
10005   ins_pipe(pipe_serial);
10006 %}
10007 
10008 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10009   predicate(n->as_LoadStore()->result_not_used());
10010   match(Set dummy (GetAndAddI mem incr));
10011   ins_cost(2 * VOLATILE_REF_COST);
10012   format %{ "get_and_addI [$mem], $incr" %}
10013   ins_encode %{
10014     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10015   %}
10016   ins_pipe(pipe_serial);
10017 %}
10018 
10019 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10020   match(Set newval (GetAndAddI mem incr));
10021   ins_cost(2 * VOLATILE_REF_COST + 1);
10022   format %{ "get_and_addI $newval, [$mem], $incr" %}
10023   ins_encode %{
10024     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10025   %}
10026   ins_pipe(pipe_serial);
10027 %}
10028 
10029 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10030   predicate(n->as_LoadStore()->result_not_used());
10031   match(Set dummy (GetAndAddI mem incr));
10032   ins_cost(2 * VOLATILE_REF_COST);
10033   format %{ "get_and_addI [$mem], $incr" %}
10034   ins_encode %{
10035     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10036   %}
10037   ins_pipe(pipe_serial);
10038 %}
10039 
10040 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10041   predicate(needs_acquiring_load_exclusive(n));
10042   match(Set newval (GetAndAddL mem incr));
10043   ins_cost(VOLATILE_REF_COST + 1);
10044   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10045   ins_encode %{
10046     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10047   %}
10048   ins_pipe(pipe_serial);
10049 %}
10050 
10051 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10052   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10053   match(Set dummy (GetAndAddL mem incr));
10054   ins_cost(VOLATILE_REF_COST);
10055   format %{ "get_and_addL_acq [$mem], $incr" %}
10056   ins_encode %{
10057     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10058   %}
10059   ins_pipe(pipe_serial);
10060 %}
10061 
10062 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10063   predicate(needs_acquiring_load_exclusive(n));
10064   match(Set newval (GetAndAddL mem incr));
10065   ins_cost(VOLATILE_REF_COST + 1);
10066   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10067   ins_encode %{
10068     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10069   %}
10070   ins_pipe(pipe_serial);
10071 %}
10072 
10073 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10074   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10075   match(Set dummy (GetAndAddL mem incr));
10076   ins_cost(VOLATILE_REF_COST);
10077   format %{ "get_and_addL_acq [$mem], $incr" %}
10078   ins_encode %{
10079     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10080   %}
10081   ins_pipe(pipe_serial);
10082 %}
10083 
10084 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10085   predicate(needs_acquiring_load_exclusive(n));
10086   match(Set newval (GetAndAddI mem incr));
10087   ins_cost(VOLATILE_REF_COST + 1);
10088   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10089   ins_encode %{
10090     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10091   %}
10092   ins_pipe(pipe_serial);
10093 %}
10094 
10095 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10096   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10097   match(Set dummy (GetAndAddI mem incr));
10098   ins_cost(VOLATILE_REF_COST);
10099   format %{ "get_and_addI_acq [$mem], $incr" %}
10100   ins_encode %{
10101     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10102   %}
10103   ins_pipe(pipe_serial);
10104 %}
10105 
10106 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10107   predicate(needs_acquiring_load_exclusive(n));
10108   match(Set newval (GetAndAddI mem incr));
10109   ins_cost(VOLATILE_REF_COST + 1);
10110   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10111   ins_encode %{
10112     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10113   %}
10114   ins_pipe(pipe_serial);
10115 %}
10116 
10117 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10118   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10119   match(Set dummy (GetAndAddI mem incr));
10120   ins_cost(VOLATILE_REF_COST);
10121   format %{ "get_and_addI_acq [$mem], $incr" %}
10122   ins_encode %{
10123     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10124   %}
10125   ins_pipe(pipe_serial);
10126 %}
10127 
10128 // Manifest a CmpL result in an integer register.
10129 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10130 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10131 %{
10132   match(Set dst (CmpL3 src1 src2));
10133   effect(KILL flags);
10134 
10135   ins_cost(INSN_COST * 6);
10136   format %{
10137       "cmp $src1, $src2"
10138       "csetw $dst, ne"
10139       "cnegw $dst, lt"
10140   %}
10141   // format %{ "CmpL3 $dst, $src1, $src2" %}
10142   ins_encode %{
10143     __ cmp($src1$$Register, $src2$$Register);
10144     __ csetw($dst$$Register, Assembler::NE);
10145     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10146   %}
10147 
10148   ins_pipe(pipe_class_default);
10149 %}
10150 
10151 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10152 %{
10153   match(Set dst (CmpL3 src1 src2));
10154   effect(KILL flags);
10155 
10156   ins_cost(INSN_COST * 6);
10157   format %{
10158       "cmp $src1, $src2"
10159       "csetw $dst, ne"
10160       "cnegw $dst, lt"
10161   %}
10162   ins_encode %{
10163     int32_t con = (int32_t)$src2$$constant;
10164      if (con < 0) {
10165       __ adds(zr, $src1$$Register, -con);
10166     } else {
10167       __ subs(zr, $src1$$Register, con);
10168     }
10169     __ csetw($dst$$Register, Assembler::NE);
10170     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10171   %}
10172 
10173   ins_pipe(pipe_class_default);
10174 %}
10175 
10176 // ============================================================================
10177 // Conditional Move Instructions
10178 
10179 // n.b. we have identical rules for both a signed compare op (cmpOp)
10180 // and an unsigned compare op (cmpOpU). it would be nice if we could
10181 // define an op class which merged both inputs and use it to type the
10182 // argument to a single rule. unfortunatelyt his fails because the
10183 // opclass does not live up to the COND_INTER interface of its
10184 // component operands. When the generic code tries to negate the
10185 // operand it ends up running the generci Machoper::negate method
10186 // which throws a ShouldNotHappen. So, we have to provide two flavours
10187 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10188 
10189 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10190   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10191 
10192   ins_cost(INSN_COST * 2);
10193   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10194 
10195   ins_encode %{
10196     __ cselw(as_Register($dst$$reg),
10197              as_Register($src2$$reg),
10198              as_Register($src1$$reg),
10199              (Assembler::Condition)$cmp$$cmpcode);
10200   %}
10201 
10202   ins_pipe(icond_reg_reg);
10203 %}
10204 
10205 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10206   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10207 
10208   ins_cost(INSN_COST * 2);
10209   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10210 
10211   ins_encode %{
10212     __ cselw(as_Register($dst$$reg),
10213              as_Register($src2$$reg),
10214              as_Register($src1$$reg),
10215              (Assembler::Condition)$cmp$$cmpcode);
10216   %}
10217 
10218   ins_pipe(icond_reg_reg);
10219 %}
10220 
10221 // special cases where one arg is zero
10222 
10223 // n.b. this is selected in preference to the rule above because it
10224 // avoids loading constant 0 into a source register
10225 
10226 // TODO
10227 // we ought only to be able to cull one of these variants as the ideal
10228 // transforms ought always to order the zero consistently (to left/right?)
10229 
10230 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10231   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10232 
10233   ins_cost(INSN_COST * 2);
10234   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10235 
10236   ins_encode %{
10237     __ cselw(as_Register($dst$$reg),
10238              as_Register($src$$reg),
10239              zr,
10240              (Assembler::Condition)$cmp$$cmpcode);
10241   %}
10242 
10243   ins_pipe(icond_reg);
10244 %}
10245 
10246 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10247   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10248 
10249   ins_cost(INSN_COST * 2);
10250   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10251 
10252   ins_encode %{
10253     __ cselw(as_Register($dst$$reg),
10254              as_Register($src$$reg),
10255              zr,
10256              (Assembler::Condition)$cmp$$cmpcode);
10257   %}
10258 
10259   ins_pipe(icond_reg);
10260 %}
10261 
10262 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10263   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10264 
10265   ins_cost(INSN_COST * 2);
10266   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10267 
10268   ins_encode %{
10269     __ cselw(as_Register($dst$$reg),
10270              zr,
10271              as_Register($src$$reg),
10272              (Assembler::Condition)$cmp$$cmpcode);
10273   %}
10274 
10275   ins_pipe(icond_reg);
10276 %}
10277 
10278 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10279   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10280 
10281   ins_cost(INSN_COST * 2);
10282   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10283 
10284   ins_encode %{
10285     __ cselw(as_Register($dst$$reg),
10286              zr,
10287              as_Register($src$$reg),
10288              (Assembler::Condition)$cmp$$cmpcode);
10289   %}
10290 
10291   ins_pipe(icond_reg);
10292 %}
10293 
10294 // special case for creating a boolean 0 or 1
10295 
10296 // n.b. this is selected in preference to the rule above because it
10297 // avoids loading constants 0 and 1 into a source register
10298 
10299 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10300   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10301 
10302   ins_cost(INSN_COST * 2);
10303   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10304 
10305   ins_encode %{
10306     // equivalently
10307     // cset(as_Register($dst$$reg),
10308     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10309     __ csincw(as_Register($dst$$reg),
10310              zr,
10311              zr,
10312              (Assembler::Condition)$cmp$$cmpcode);
10313   %}
10314 
10315   ins_pipe(icond_none);
10316 %}
10317 
10318 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10319   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10320 
10321   ins_cost(INSN_COST * 2);
10322   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10323 
10324   ins_encode %{
10325     // equivalently
10326     // cset(as_Register($dst$$reg),
10327     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10328     __ csincw(as_Register($dst$$reg),
10329              zr,
10330              zr,
10331              (Assembler::Condition)$cmp$$cmpcode);
10332   %}
10333 
10334   ins_pipe(icond_none);
10335 %}
10336 
10337 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10338   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10339 
10340   ins_cost(INSN_COST * 2);
10341   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10342 
10343   ins_encode %{
10344     __ csel(as_Register($dst$$reg),
10345             as_Register($src2$$reg),
10346             as_Register($src1$$reg),
10347             (Assembler::Condition)$cmp$$cmpcode);
10348   %}
10349 
10350   ins_pipe(icond_reg_reg);
10351 %}
10352 
10353 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10354   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10355 
10356   ins_cost(INSN_COST * 2);
10357   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10358 
10359   ins_encode %{
10360     __ csel(as_Register($dst$$reg),
10361             as_Register($src2$$reg),
10362             as_Register($src1$$reg),
10363             (Assembler::Condition)$cmp$$cmpcode);
10364   %}
10365 
10366   ins_pipe(icond_reg_reg);
10367 %}
10368 
10369 // special cases where one arg is zero
10370 
10371 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10372   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10373 
10374   ins_cost(INSN_COST * 2);
10375   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10376 
10377   ins_encode %{
10378     __ csel(as_Register($dst$$reg),
10379             zr,
10380             as_Register($src$$reg),
10381             (Assembler::Condition)$cmp$$cmpcode);
10382   %}
10383 
10384   ins_pipe(icond_reg);
10385 %}
10386 
10387 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10388   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10389 
10390   ins_cost(INSN_COST * 2);
10391   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10392 
10393   ins_encode %{
10394     __ csel(as_Register($dst$$reg),
10395             zr,
10396             as_Register($src$$reg),
10397             (Assembler::Condition)$cmp$$cmpcode);
10398   %}
10399 
10400   ins_pipe(icond_reg);
10401 %}
10402 
10403 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10404   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10405 
10406   ins_cost(INSN_COST * 2);
10407   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10408 
10409   ins_encode %{
10410     __ csel(as_Register($dst$$reg),
10411             as_Register($src$$reg),
10412             zr,
10413             (Assembler::Condition)$cmp$$cmpcode);
10414   %}
10415 
10416   ins_pipe(icond_reg);
10417 %}
10418 
10419 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10420   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10421 
10422   ins_cost(INSN_COST * 2);
10423   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10424 
10425   ins_encode %{
10426     __ csel(as_Register($dst$$reg),
10427             as_Register($src$$reg),
10428             zr,
10429             (Assembler::Condition)$cmp$$cmpcode);
10430   %}
10431 
10432   ins_pipe(icond_reg);
10433 %}
10434 
10435 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10436   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10437 
10438   ins_cost(INSN_COST * 2);
10439   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10440 
10441   ins_encode %{
10442     __ csel(as_Register($dst$$reg),
10443             as_Register($src2$$reg),
10444             as_Register($src1$$reg),
10445             (Assembler::Condition)$cmp$$cmpcode);
10446   %}
10447 
10448   ins_pipe(icond_reg_reg);
10449 %}
10450 
10451 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10452   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10453 
10454   ins_cost(INSN_COST * 2);
10455   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10456 
10457   ins_encode %{
10458     __ csel(as_Register($dst$$reg),
10459             as_Register($src2$$reg),
10460             as_Register($src1$$reg),
10461             (Assembler::Condition)$cmp$$cmpcode);
10462   %}
10463 
10464   ins_pipe(icond_reg_reg);
10465 %}
10466 
10467 // special cases where one arg is zero
10468 
10469 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10470   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10471 
10472   ins_cost(INSN_COST * 2);
10473   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10474 
10475   ins_encode %{
10476     __ csel(as_Register($dst$$reg),
10477             zr,
10478             as_Register($src$$reg),
10479             (Assembler::Condition)$cmp$$cmpcode);
10480   %}
10481 
10482   ins_pipe(icond_reg);
10483 %}
10484 
10485 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10486   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10487 
10488   ins_cost(INSN_COST * 2);
10489   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10490 
10491   ins_encode %{
10492     __ csel(as_Register($dst$$reg),
10493             zr,
10494             as_Register($src$$reg),
10495             (Assembler::Condition)$cmp$$cmpcode);
10496   %}
10497 
10498   ins_pipe(icond_reg);
10499 %}
10500 
10501 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10502   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10503 
10504   ins_cost(INSN_COST * 2);
10505   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10506 
10507   ins_encode %{
10508     __ csel(as_Register($dst$$reg),
10509             as_Register($src$$reg),
10510             zr,
10511             (Assembler::Condition)$cmp$$cmpcode);
10512   %}
10513 
10514   ins_pipe(icond_reg);
10515 %}
10516 
10517 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10518   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10519 
10520   ins_cost(INSN_COST * 2);
10521   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10522 
10523   ins_encode %{
10524     __ csel(as_Register($dst$$reg),
10525             as_Register($src$$reg),
10526             zr,
10527             (Assembler::Condition)$cmp$$cmpcode);
10528   %}
10529 
10530   ins_pipe(icond_reg);
10531 %}
10532 
10533 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10534   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10535 
10536   ins_cost(INSN_COST * 2);
10537   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10538 
10539   ins_encode %{
10540     __ cselw(as_Register($dst$$reg),
10541              as_Register($src2$$reg),
10542              as_Register($src1$$reg),
10543              (Assembler::Condition)$cmp$$cmpcode);
10544   %}
10545 
10546   ins_pipe(icond_reg_reg);
10547 %}
10548 
10549 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10550   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10551 
10552   ins_cost(INSN_COST * 2);
10553   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10554 
10555   ins_encode %{
10556     __ cselw(as_Register($dst$$reg),
10557              as_Register($src2$$reg),
10558              as_Register($src1$$reg),
10559              (Assembler::Condition)$cmp$$cmpcode);
10560   %}
10561 
10562   ins_pipe(icond_reg_reg);
10563 %}
10564 
10565 // special cases where one arg is zero
10566 
10567 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10568   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10569 
10570   ins_cost(INSN_COST * 2);
10571   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10572 
10573   ins_encode %{
10574     __ cselw(as_Register($dst$$reg),
10575              zr,
10576              as_Register($src$$reg),
10577              (Assembler::Condition)$cmp$$cmpcode);
10578   %}
10579 
10580   ins_pipe(icond_reg);
10581 %}
10582 
10583 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10584   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10585 
10586   ins_cost(INSN_COST * 2);
10587   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10588 
10589   ins_encode %{
10590     __ cselw(as_Register($dst$$reg),
10591              zr,
10592              as_Register($src$$reg),
10593              (Assembler::Condition)$cmp$$cmpcode);
10594   %}
10595 
10596   ins_pipe(icond_reg);
10597 %}
10598 
10599 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10600   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10601 
10602   ins_cost(INSN_COST * 2);
10603   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10604 
10605   ins_encode %{
10606     __ cselw(as_Register($dst$$reg),
10607              as_Register($src$$reg),
10608              zr,
10609              (Assembler::Condition)$cmp$$cmpcode);
10610   %}
10611 
10612   ins_pipe(icond_reg);
10613 %}
10614 
10615 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10616   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10617 
10618   ins_cost(INSN_COST * 2);
10619   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10620 
10621   ins_encode %{
10622     __ cselw(as_Register($dst$$reg),
10623              as_Register($src$$reg),
10624              zr,
10625              (Assembler::Condition)$cmp$$cmpcode);
10626   %}
10627 
10628   ins_pipe(icond_reg);
10629 %}
10630 
10631 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10632 %{
10633   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10634 
10635   ins_cost(INSN_COST * 3);
10636 
10637   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10638   ins_encode %{
10639     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10640     __ fcsels(as_FloatRegister($dst$$reg),
10641               as_FloatRegister($src2$$reg),
10642               as_FloatRegister($src1$$reg),
10643               cond);
10644   %}
10645 
10646   ins_pipe(fp_cond_reg_reg_s);
10647 %}
10648 
10649 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10650 %{
10651   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10652 
10653   ins_cost(INSN_COST * 3);
10654 
10655   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10656   ins_encode %{
10657     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10658     __ fcsels(as_FloatRegister($dst$$reg),
10659               as_FloatRegister($src2$$reg),
10660               as_FloatRegister($src1$$reg),
10661               cond);
10662   %}
10663 
10664   ins_pipe(fp_cond_reg_reg_s);
10665 %}
10666 
10667 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10668 %{
10669   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10670 
10671   ins_cost(INSN_COST * 3);
10672 
10673   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10674   ins_encode %{
10675     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10676     __ fcseld(as_FloatRegister($dst$$reg),
10677               as_FloatRegister($src2$$reg),
10678               as_FloatRegister($src1$$reg),
10679               cond);
10680   %}
10681 
10682   ins_pipe(fp_cond_reg_reg_d);
10683 %}
10684 
10685 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10686 %{
10687   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10688 
10689   ins_cost(INSN_COST * 3);
10690 
10691   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10692   ins_encode %{
10693     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10694     __ fcseld(as_FloatRegister($dst$$reg),
10695               as_FloatRegister($src2$$reg),
10696               as_FloatRegister($src1$$reg),
10697               cond);
10698   %}
10699 
10700   ins_pipe(fp_cond_reg_reg_d);
10701 %}
10702 
10703 // ============================================================================
10704 // Arithmetic Instructions
10705 //
10706 
10707 // Integer Addition
10708 
10709 // TODO
10710 // these currently employ operations which do not set CR and hence are
10711 // not flagged as killing CR but we would like to isolate the cases
10712 // where we want to set flags from those where we don't. need to work
10713 // out how to do that.
10714 
10715 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10716   match(Set dst (AddI src1 src2));
10717 
10718   ins_cost(INSN_COST);
10719   format %{ "addw  $dst, $src1, $src2" %}
10720 
10721   ins_encode %{
10722     __ addw(as_Register($dst$$reg),
10723             as_Register($src1$$reg),
10724             as_Register($src2$$reg));
10725   %}
10726 
10727   ins_pipe(ialu_reg_reg);
10728 %}
10729 
10730 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10731   match(Set dst (AddI src1 src2));
10732 
10733   ins_cost(INSN_COST);
10734   format %{ "addw $dst, $src1, $src2" %}
10735 
10736   // use opcode to indicate that this is an add not a sub
10737   opcode(0x0);
10738 
10739   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10740 
10741   ins_pipe(ialu_reg_imm);
10742 %}
10743 
10744 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10745   match(Set dst (AddI (ConvL2I src1) src2));
10746 
10747   ins_cost(INSN_COST);
10748   format %{ "addw $dst, $src1, $src2" %}
10749 
10750   // use opcode to indicate that this is an add not a sub
10751   opcode(0x0);
10752 
10753   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10754 
10755   ins_pipe(ialu_reg_imm);
10756 %}
10757 
10758 // Pointer Addition
10759 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10760   match(Set dst (AddP src1 src2));
10761 
10762   ins_cost(INSN_COST);
10763   format %{ "add $dst, $src1, $src2\t# ptr" %}
10764 
10765   ins_encode %{
10766     __ add(as_Register($dst$$reg),
10767            as_Register($src1$$reg),
10768            as_Register($src2$$reg));
10769   %}
10770 
10771   ins_pipe(ialu_reg_reg);
10772 %}
10773 
10774 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10775   match(Set dst (AddP src1 (ConvI2L src2)));
10776 
10777   ins_cost(1.9 * INSN_COST);
10778   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10779 
10780   ins_encode %{
10781     __ add(as_Register($dst$$reg),
10782            as_Register($src1$$reg),
10783            as_Register($src2$$reg), ext::sxtw);
10784   %}
10785 
10786   ins_pipe(ialu_reg_reg);
10787 %}
10788 
10789 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10790   match(Set dst (AddP src1 (LShiftL src2 scale)));
10791 
10792   ins_cost(1.9 * INSN_COST);
10793   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10794 
10795   ins_encode %{
10796     __ lea(as_Register($dst$$reg),
10797            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10798                    Address::lsl($scale$$constant)));
10799   %}
10800 
10801   ins_pipe(ialu_reg_reg_shift);
10802 %}
10803 
10804 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10805   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10806 
10807   ins_cost(1.9 * INSN_COST);
10808   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10809 
10810   ins_encode %{
10811     __ lea(as_Register($dst$$reg),
10812            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10813                    Address::sxtw($scale$$constant)));
10814   %}
10815 
10816   ins_pipe(ialu_reg_reg_shift);
10817 %}
10818 
10819 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10820   match(Set dst (LShiftL (ConvI2L src) scale));
10821 
10822   ins_cost(INSN_COST);
10823   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10824 
10825   ins_encode %{
10826     __ sbfiz(as_Register($dst$$reg),
10827           as_Register($src$$reg),
10828           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10829   %}
10830 
10831   ins_pipe(ialu_reg_shift);
10832 %}
10833 
10834 // Pointer Immediate Addition
10835 // n.b. this needs to be more expensive than using an indirect memory
10836 // operand
10837 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10838   match(Set dst (AddP src1 src2));
10839 
10840   ins_cost(INSN_COST);
10841   format %{ "add $dst, $src1, $src2\t# ptr" %}
10842 
10843   // use opcode to indicate that this is an add not a sub
10844   opcode(0x0);
10845 
10846   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10847 
10848   ins_pipe(ialu_reg_imm);
10849 %}
10850 
10851 // Long Addition
10852 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10853 
10854   match(Set dst (AddL src1 src2));
10855 
10856   ins_cost(INSN_COST);
10857   format %{ "add  $dst, $src1, $src2" %}
10858 
10859   ins_encode %{
10860     __ add(as_Register($dst$$reg),
10861            as_Register($src1$$reg),
10862            as_Register($src2$$reg));
10863   %}
10864 
10865   ins_pipe(ialu_reg_reg);
10866 %}
10867 
10868 // No constant pool entries requiredLong Immediate Addition.
10869 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10870   match(Set dst (AddL src1 src2));
10871 
10872   ins_cost(INSN_COST);
10873   format %{ "add $dst, $src1, $src2" %}
10874 
10875   // use opcode to indicate that this is an add not a sub
10876   opcode(0x0);
10877 
10878   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10879 
10880   ins_pipe(ialu_reg_imm);
10881 %}
10882 
10883 // Integer Subtraction
10884 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10885   match(Set dst (SubI src1 src2));
10886 
10887   ins_cost(INSN_COST);
10888   format %{ "subw  $dst, $src1, $src2" %}
10889 
10890   ins_encode %{
10891     __ subw(as_Register($dst$$reg),
10892             as_Register($src1$$reg),
10893             as_Register($src2$$reg));
10894   %}
10895 
10896   ins_pipe(ialu_reg_reg);
10897 %}
10898 
10899 // Immediate Subtraction
10900 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10901   match(Set dst (SubI src1 src2));
10902 
10903   ins_cost(INSN_COST);
10904   format %{ "subw $dst, $src1, $src2" %}
10905 
10906   // use opcode to indicate that this is a sub not an add
10907   opcode(0x1);
10908 
10909   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10910 
10911   ins_pipe(ialu_reg_imm);
10912 %}
10913 
10914 // Long Subtraction
10915 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10916 
10917   match(Set dst (SubL src1 src2));
10918 
10919   ins_cost(INSN_COST);
10920   format %{ "sub  $dst, $src1, $src2" %}
10921 
10922   ins_encode %{
10923     __ sub(as_Register($dst$$reg),
10924            as_Register($src1$$reg),
10925            as_Register($src2$$reg));
10926   %}
10927 
10928   ins_pipe(ialu_reg_reg);
10929 %}
10930 
10931 // No constant pool entries requiredLong Immediate Subtraction.
10932 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10933   match(Set dst (SubL src1 src2));
10934 
10935   ins_cost(INSN_COST);
10936   format %{ "sub$dst, $src1, $src2" %}
10937 
10938   // use opcode to indicate that this is a sub not an add
10939   opcode(0x1);
10940 
10941   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10942 
10943   ins_pipe(ialu_reg_imm);
10944 %}
10945 
10946 // Integer Negation (special case for sub)
10947 
10948 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10949   match(Set dst (SubI zero src));
10950 
10951   ins_cost(INSN_COST);
10952   format %{ "negw $dst, $src\t# int" %}
10953 
10954   ins_encode %{
10955     __ negw(as_Register($dst$$reg),
10956             as_Register($src$$reg));
10957   %}
10958 
10959   ins_pipe(ialu_reg);
10960 %}
10961 
10962 // Long Negation
10963 
10964 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10965   match(Set dst (SubL zero src));
10966 
10967   ins_cost(INSN_COST);
10968   format %{ "neg $dst, $src\t# long" %}
10969 
10970   ins_encode %{
10971     __ neg(as_Register($dst$$reg),
10972            as_Register($src$$reg));
10973   %}
10974 
10975   ins_pipe(ialu_reg);
10976 %}
10977 
10978 // Integer Multiply
10979 
10980 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10981   match(Set dst (MulI src1 src2));
10982 
10983   ins_cost(INSN_COST * 3);
10984   format %{ "mulw  $dst, $src1, $src2" %}
10985 
10986   ins_encode %{
10987     __ mulw(as_Register($dst$$reg),
10988             as_Register($src1$$reg),
10989             as_Register($src2$$reg));
10990   %}
10991 
10992   ins_pipe(imul_reg_reg);
10993 %}
10994 
10995 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10996   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10997 
10998   ins_cost(INSN_COST * 3);
10999   format %{ "smull  $dst, $src1, $src2" %}
11000 
11001   ins_encode %{
11002     __ smull(as_Register($dst$$reg),
11003              as_Register($src1$$reg),
11004              as_Register($src2$$reg));
11005   %}
11006 
11007   ins_pipe(imul_reg_reg);
11008 %}
11009 
11010 // Long Multiply
11011 
11012 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11013   match(Set dst (MulL src1 src2));
11014 
11015   ins_cost(INSN_COST * 5);
11016   format %{ "mul  $dst, $src1, $src2" %}
11017 
11018   ins_encode %{
11019     __ mul(as_Register($dst$$reg),
11020            as_Register($src1$$reg),
11021            as_Register($src2$$reg));
11022   %}
11023 
11024   ins_pipe(lmul_reg_reg);
11025 %}
11026 
11027 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11028 %{
11029   match(Set dst (MulHiL src1 src2));
11030 
11031   ins_cost(INSN_COST * 7);
11032   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
11033 
11034   ins_encode %{
11035     __ smulh(as_Register($dst$$reg),
11036              as_Register($src1$$reg),
11037              as_Register($src2$$reg));
11038   %}
11039 
11040   ins_pipe(lmul_reg_reg);
11041 %}
11042 
11043 // Combined Integer Multiply & Add/Sub
11044 
11045 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11046   match(Set dst (AddI src3 (MulI src1 src2)));
11047 
11048   ins_cost(INSN_COST * 3);
11049   format %{ "madd  $dst, $src1, $src2, $src3" %}
11050 
11051   ins_encode %{
11052     __ maddw(as_Register($dst$$reg),
11053              as_Register($src1$$reg),
11054              as_Register($src2$$reg),
11055              as_Register($src3$$reg));
11056   %}
11057 
11058   ins_pipe(imac_reg_reg);
11059 %}
11060 
11061 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11062   match(Set dst (SubI src3 (MulI src1 src2)));
11063 
11064   ins_cost(INSN_COST * 3);
11065   format %{ "msub  $dst, $src1, $src2, $src3" %}
11066 
11067   ins_encode %{
11068     __ msubw(as_Register($dst$$reg),
11069              as_Register($src1$$reg),
11070              as_Register($src2$$reg),
11071              as_Register($src3$$reg));
11072   %}
11073 
11074   ins_pipe(imac_reg_reg);
11075 %}
11076 
11077 // Combined Integer Multiply & Neg
11078 
11079 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11080   match(Set dst (MulI (SubI zero src1) src2));
11081 
11082   ins_cost(INSN_COST * 3);
11083   format %{ "mneg  $dst, $src1, $src2" %}
11084 
11085   ins_encode %{
11086     __ mnegw(as_Register($dst$$reg),
11087              as_Register($src1$$reg),
11088              as_Register($src2$$reg));
11089   %}
11090 
11091   ins_pipe(imac_reg_reg);
11092 %}
11093 
11094 // Combined Long Multiply & Add/Sub
11095 
11096 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11097   match(Set dst (AddL src3 (MulL src1 src2)));
11098 
11099   ins_cost(INSN_COST * 5);
11100   format %{ "madd  $dst, $src1, $src2, $src3" %}
11101 
11102   ins_encode %{
11103     __ madd(as_Register($dst$$reg),
11104             as_Register($src1$$reg),
11105             as_Register($src2$$reg),
11106             as_Register($src3$$reg));
11107   %}
11108 
11109   ins_pipe(lmac_reg_reg);
11110 %}
11111 
11112 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11113   match(Set dst (SubL src3 (MulL src1 src2)));
11114 
11115   ins_cost(INSN_COST * 5);
11116   format %{ "msub  $dst, $src1, $src2, $src3" %}
11117 
11118   ins_encode %{
11119     __ msub(as_Register($dst$$reg),
11120             as_Register($src1$$reg),
11121             as_Register($src2$$reg),
11122             as_Register($src3$$reg));
11123   %}
11124 
11125   ins_pipe(lmac_reg_reg);
11126 %}
11127 
11128 // Combined Long Multiply & Neg
11129 
11130 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11131   match(Set dst (MulL (SubL zero src1) src2));
11132 
11133   ins_cost(INSN_COST * 5);
11134   format %{ "mneg  $dst, $src1, $src2" %}
11135 
11136   ins_encode %{
11137     __ mneg(as_Register($dst$$reg),
11138             as_Register($src1$$reg),
11139             as_Register($src2$$reg));
11140   %}
11141 
11142   ins_pipe(lmac_reg_reg);
11143 %}
11144 
11145 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11146 
11147 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11148   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11149 
11150   ins_cost(INSN_COST * 3);
11151   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11152 
11153   ins_encode %{
11154     __ smaddl(as_Register($dst$$reg),
11155               as_Register($src1$$reg),
11156               as_Register($src2$$reg),
11157               as_Register($src3$$reg));
11158   %}
11159 
11160   ins_pipe(imac_reg_reg);
11161 %}
11162 
11163 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11164   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11165 
11166   ins_cost(INSN_COST * 3);
11167   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11168 
11169   ins_encode %{
11170     __ smsubl(as_Register($dst$$reg),
11171               as_Register($src1$$reg),
11172               as_Register($src2$$reg),
11173               as_Register($src3$$reg));
11174   %}
11175 
11176   ins_pipe(imac_reg_reg);
11177 %}
11178 
11179 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11180   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11181 
11182   ins_cost(INSN_COST * 3);
11183   format %{ "smnegl  $dst, $src1, $src2" %}
11184 
11185   ins_encode %{
11186     __ smnegl(as_Register($dst$$reg),
11187               as_Register($src1$$reg),
11188               as_Register($src2$$reg));
11189   %}
11190 
11191   ins_pipe(imac_reg_reg);
11192 %}
11193 
11194 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11195 
11196 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11197   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11198 
11199   ins_cost(INSN_COST * 5);
11200   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11201             "maddw $dst, $src3, $src4, rscratch1" %}
11202 
11203   ins_encode %{
11204     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11205     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11206 
11207   ins_pipe(imac_reg_reg);
11208 %}
11209 
11210 // Integer Divide
11211 
11212 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11213   match(Set dst (DivI src1 src2));
11214 
11215   ins_cost(INSN_COST * 19);
11216   format %{ "sdivw  $dst, $src1, $src2" %}
11217 
11218   ins_encode(aarch64_enc_divw(dst, src1, src2));
11219   ins_pipe(idiv_reg_reg);
11220 %}
11221 
11222 // Long Divide
11223 
11224 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11225   match(Set dst (DivL src1 src2));
11226 
11227   ins_cost(INSN_COST * 35);
11228   format %{ "sdiv   $dst, $src1, $src2" %}
11229 
11230   ins_encode(aarch64_enc_div(dst, src1, src2));
11231   ins_pipe(ldiv_reg_reg);
11232 %}
11233 
11234 // Integer Remainder
11235 
11236 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11237   match(Set dst (ModI src1 src2));
11238 
11239   ins_cost(INSN_COST * 22);
11240   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11241             "msubw($dst, rscratch1, $src2, $src1" %}
11242 
11243   ins_encode(aarch64_enc_modw(dst, src1, src2));
11244   ins_pipe(idiv_reg_reg);
11245 %}
11246 
11247 // Long Remainder
11248 
11249 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11250   match(Set dst (ModL src1 src2));
11251 
11252   ins_cost(INSN_COST * 38);
11253   format %{ "sdiv   rscratch1, $src1, $src2\n"
11254             "msub($dst, rscratch1, $src2, $src1" %}
11255 
11256   ins_encode(aarch64_enc_mod(dst, src1, src2));
11257   ins_pipe(ldiv_reg_reg);
11258 %}
11259 
11260 // Integer Shifts
11261 
11262 // Shift Left Register
11263 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11264   match(Set dst (LShiftI src1 src2));
11265 
11266   ins_cost(INSN_COST * 2);
11267   format %{ "lslvw  $dst, $src1, $src2" %}
11268 
11269   ins_encode %{
11270     __ lslvw(as_Register($dst$$reg),
11271              as_Register($src1$$reg),
11272              as_Register($src2$$reg));
11273   %}
11274 
11275   ins_pipe(ialu_reg_reg_vshift);
11276 %}
11277 
11278 // Shift Left Immediate
11279 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11280   match(Set dst (LShiftI src1 src2));
11281 
11282   ins_cost(INSN_COST);
11283   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11284 
11285   ins_encode %{
11286     __ lslw(as_Register($dst$$reg),
11287             as_Register($src1$$reg),
11288             $src2$$constant & 0x1f);
11289   %}
11290 
11291   ins_pipe(ialu_reg_shift);
11292 %}
11293 
11294 // Shift Right Logical Register
11295 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11296   match(Set dst (URShiftI src1 src2));
11297 
11298   ins_cost(INSN_COST * 2);
11299   format %{ "lsrvw  $dst, $src1, $src2" %}
11300 
11301   ins_encode %{
11302     __ lsrvw(as_Register($dst$$reg),
11303              as_Register($src1$$reg),
11304              as_Register($src2$$reg));
11305   %}
11306 
11307   ins_pipe(ialu_reg_reg_vshift);
11308 %}
11309 
11310 // Shift Right Logical Immediate
11311 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11312   match(Set dst (URShiftI src1 src2));
11313 
11314   ins_cost(INSN_COST);
11315   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11316 
11317   ins_encode %{
11318     __ lsrw(as_Register($dst$$reg),
11319             as_Register($src1$$reg),
11320             $src2$$constant & 0x1f);
11321   %}
11322 
11323   ins_pipe(ialu_reg_shift);
11324 %}
11325 
11326 // Shift Right Arithmetic Register
11327 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11328   match(Set dst (RShiftI src1 src2));
11329 
11330   ins_cost(INSN_COST * 2);
11331   format %{ "asrvw  $dst, $src1, $src2" %}
11332 
11333   ins_encode %{
11334     __ asrvw(as_Register($dst$$reg),
11335              as_Register($src1$$reg),
11336              as_Register($src2$$reg));
11337   %}
11338 
11339   ins_pipe(ialu_reg_reg_vshift);
11340 %}
11341 
11342 // Shift Right Arithmetic Immediate
11343 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11344   match(Set dst (RShiftI src1 src2));
11345 
11346   ins_cost(INSN_COST);
11347   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11348 
11349   ins_encode %{
11350     __ asrw(as_Register($dst$$reg),
11351             as_Register($src1$$reg),
11352             $src2$$constant & 0x1f);
11353   %}
11354 
11355   ins_pipe(ialu_reg_shift);
11356 %}
11357 
11358 // Combined Int Mask and Right Shift (using UBFM)
11359 // TODO
11360 
11361 // Long Shifts
11362 
11363 // Shift Left Register
11364 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11365   match(Set dst (LShiftL src1 src2));
11366 
11367   ins_cost(INSN_COST * 2);
11368   format %{ "lslv  $dst, $src1, $src2" %}
11369 
11370   ins_encode %{
11371     __ lslv(as_Register($dst$$reg),
11372             as_Register($src1$$reg),
11373             as_Register($src2$$reg));
11374   %}
11375 
11376   ins_pipe(ialu_reg_reg_vshift);
11377 %}
11378 
11379 // Shift Left Immediate
11380 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11381   match(Set dst (LShiftL src1 src2));
11382 
11383   ins_cost(INSN_COST);
11384   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11385 
11386   ins_encode %{
11387     __ lsl(as_Register($dst$$reg),
11388             as_Register($src1$$reg),
11389             $src2$$constant & 0x3f);
11390   %}
11391 
11392   ins_pipe(ialu_reg_shift);
11393 %}
11394 
11395 // Shift Right Logical Register
11396 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11397   match(Set dst (URShiftL src1 src2));
11398 
11399   ins_cost(INSN_COST * 2);
11400   format %{ "lsrv  $dst, $src1, $src2" %}
11401 
11402   ins_encode %{
11403     __ lsrv(as_Register($dst$$reg),
11404             as_Register($src1$$reg),
11405             as_Register($src2$$reg));
11406   %}
11407 
11408   ins_pipe(ialu_reg_reg_vshift);
11409 %}
11410 
11411 // Shift Right Logical Immediate
11412 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11413   match(Set dst (URShiftL src1 src2));
11414 
11415   ins_cost(INSN_COST);
11416   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11417 
11418   ins_encode %{
11419     __ lsr(as_Register($dst$$reg),
11420            as_Register($src1$$reg),
11421            $src2$$constant & 0x3f);
11422   %}
11423 
11424   ins_pipe(ialu_reg_shift);
11425 %}
11426 
11427 // A special-case pattern for card table stores.
11428 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11429   match(Set dst (URShiftL (CastP2X src1) src2));
11430 
11431   ins_cost(INSN_COST);
11432   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11433 
11434   ins_encode %{
11435     __ lsr(as_Register($dst$$reg),
11436            as_Register($src1$$reg),
11437            $src2$$constant & 0x3f);
11438   %}
11439 
11440   ins_pipe(ialu_reg_shift);
11441 %}
11442 
11443 // Shift Right Arithmetic Register
11444 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11445   match(Set dst (RShiftL src1 src2));
11446 
11447   ins_cost(INSN_COST * 2);
11448   format %{ "asrv  $dst, $src1, $src2" %}
11449 
11450   ins_encode %{
11451     __ asrv(as_Register($dst$$reg),
11452             as_Register($src1$$reg),
11453             as_Register($src2$$reg));
11454   %}
11455 
11456   ins_pipe(ialu_reg_reg_vshift);
11457 %}
11458 
11459 // Shift Right Arithmetic Immediate
11460 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11461   match(Set dst (RShiftL src1 src2));
11462 
11463   ins_cost(INSN_COST);
11464   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11465 
11466   ins_encode %{
11467     __ asr(as_Register($dst$$reg),
11468            as_Register($src1$$reg),
11469            $src2$$constant & 0x3f);
11470   %}
11471 
11472   ins_pipe(ialu_reg_shift);
11473 %}
11474 
11475 // BEGIN This section of the file is automatically generated. Do not edit --------------
11476 // This section is generated from aarch64_ad.m4
11477 
11478 
11479 // This pattern is automatically generated from aarch64_ad.m4
11480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11481 instruct regL_not_reg(iRegLNoSp dst,
11482                          iRegL src1, immL_M1 m1,
11483                          rFlagsReg cr) %{
11484   match(Set dst (XorL src1 m1));
11485   ins_cost(INSN_COST);
11486   format %{ "eon  $dst, $src1, zr" %}
11487 
11488   ins_encode %{
11489     __ eon(as_Register($dst$$reg),
11490               as_Register($src1$$reg),
11491               zr,
11492               Assembler::LSL, 0);
11493   %}
11494 
11495   ins_pipe(ialu_reg);
11496 %}
11497 
11498 // This pattern is automatically generated from aarch64_ad.m4
11499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11500 instruct regI_not_reg(iRegINoSp dst,
11501                          iRegIorL2I src1, immI_M1 m1,
11502                          rFlagsReg cr) %{
11503   match(Set dst (XorI src1 m1));
11504   ins_cost(INSN_COST);
11505   format %{ "eonw  $dst, $src1, zr" %}
11506 
11507   ins_encode %{
11508     __ eonw(as_Register($dst$$reg),
11509               as_Register($src1$$reg),
11510               zr,
11511               Assembler::LSL, 0);
11512   %}
11513 
11514   ins_pipe(ialu_reg);
11515 %}
11516 
11517 // This pattern is automatically generated from aarch64_ad.m4
11518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11519 instruct AndI_reg_not_reg(iRegINoSp dst,
11520                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11521   match(Set dst (AndI src1 (XorI src2 m1)));
11522   ins_cost(INSN_COST);
11523   format %{ "bicw  $dst, $src1, $src2" %}
11524 
11525   ins_encode %{
11526     __ bicw(as_Register($dst$$reg),
11527               as_Register($src1$$reg),
11528               as_Register($src2$$reg),
11529               Assembler::LSL, 0);
11530   %}
11531 
11532   ins_pipe(ialu_reg_reg);
11533 %}
11534 
11535 // This pattern is automatically generated from aarch64_ad.m4
11536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11537 instruct AndL_reg_not_reg(iRegLNoSp dst,
11538                          iRegL src1, iRegL src2, immL_M1 m1) %{
11539   match(Set dst (AndL src1 (XorL src2 m1)));
11540   ins_cost(INSN_COST);
11541   format %{ "bic  $dst, $src1, $src2" %}
11542 
11543   ins_encode %{
11544     __ bic(as_Register($dst$$reg),
11545               as_Register($src1$$reg),
11546               as_Register($src2$$reg),
11547               Assembler::LSL, 0);
11548   %}
11549 
11550   ins_pipe(ialu_reg_reg);
11551 %}
11552 
11553 // This pattern is automatically generated from aarch64_ad.m4
11554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11555 instruct OrI_reg_not_reg(iRegINoSp dst,
11556                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11557   match(Set dst (OrI src1 (XorI src2 m1)));
11558   ins_cost(INSN_COST);
11559   format %{ "ornw  $dst, $src1, $src2" %}
11560 
11561   ins_encode %{
11562     __ ornw(as_Register($dst$$reg),
11563               as_Register($src1$$reg),
11564               as_Register($src2$$reg),
11565               Assembler::LSL, 0);
11566   %}
11567 
11568   ins_pipe(ialu_reg_reg);
11569 %}
11570 
11571 // This pattern is automatically generated from aarch64_ad.m4
11572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11573 instruct OrL_reg_not_reg(iRegLNoSp dst,
11574                          iRegL src1, iRegL src2, immL_M1 m1) %{
11575   match(Set dst (OrL src1 (XorL src2 m1)));
11576   ins_cost(INSN_COST);
11577   format %{ "orn  $dst, $src1, $src2" %}
11578 
11579   ins_encode %{
11580     __ orn(as_Register($dst$$reg),
11581               as_Register($src1$$reg),
11582               as_Register($src2$$reg),
11583               Assembler::LSL, 0);
11584   %}
11585 
11586   ins_pipe(ialu_reg_reg);
11587 %}
11588 
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591 instruct XorI_reg_not_reg(iRegINoSp dst,
11592                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11593   match(Set dst (XorI m1 (XorI src2 src1)));
11594   ins_cost(INSN_COST);
11595   format %{ "eonw  $dst, $src1, $src2" %}
11596 
11597   ins_encode %{
11598     __ eonw(as_Register($dst$$reg),
11599               as_Register($src1$$reg),
11600               as_Register($src2$$reg),
11601               Assembler::LSL, 0);
11602   %}
11603 
11604   ins_pipe(ialu_reg_reg);
11605 %}
11606 
11607 // This pattern is automatically generated from aarch64_ad.m4
11608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11609 instruct XorL_reg_not_reg(iRegLNoSp dst,
11610                          iRegL src1, iRegL src2, immL_M1 m1) %{
11611   match(Set dst (XorL m1 (XorL src2 src1)));
11612   ins_cost(INSN_COST);
11613   format %{ "eon  $dst, $src1, $src2" %}
11614 
11615   ins_encode %{
11616     __ eon(as_Register($dst$$reg),
11617               as_Register($src1$$reg),
11618               as_Register($src2$$reg),
11619               Assembler::LSL, 0);
11620   %}
11621 
11622   ins_pipe(ialu_reg_reg);
11623 %}
11624 
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 // val & (-1 ^ (val >>> shift)) ==> bicw
11628 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11629                          iRegIorL2I src1, iRegIorL2I src2,
11630                          immI src3, immI_M1 src4) %{
11631   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11632   ins_cost(1.9 * INSN_COST);
11633   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11634 
11635   ins_encode %{
11636     __ bicw(as_Register($dst$$reg),
11637               as_Register($src1$$reg),
11638               as_Register($src2$$reg),
11639               Assembler::LSR,
11640               $src3$$constant & 0x1f);
11641   %}
11642 
11643   ins_pipe(ialu_reg_reg_shift);
11644 %}
11645 
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648 // val & (-1 ^ (val >>> shift)) ==> bic
11649 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11650                          iRegL src1, iRegL src2,
11651                          immI src3, immL_M1 src4) %{
11652   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11653   ins_cost(1.9 * INSN_COST);
11654   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11655 
11656   ins_encode %{
11657     __ bic(as_Register($dst$$reg),
11658               as_Register($src1$$reg),
11659               as_Register($src2$$reg),
11660               Assembler::LSR,
11661               $src3$$constant & 0x3f);
11662   %}
11663 
11664   ins_pipe(ialu_reg_reg_shift);
11665 %}
11666 
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669 // val & (-1 ^ (val >> shift)) ==> bicw
11670 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11671                          iRegIorL2I src1, iRegIorL2I src2,
11672                          immI src3, immI_M1 src4) %{
11673   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11674   ins_cost(1.9 * INSN_COST);
11675   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11676 
11677   ins_encode %{
11678     __ bicw(as_Register($dst$$reg),
11679               as_Register($src1$$reg),
11680               as_Register($src2$$reg),
11681               Assembler::ASR,
11682               $src3$$constant & 0x1f);
11683   %}
11684 
11685   ins_pipe(ialu_reg_reg_shift);
11686 %}
11687 
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 // val & (-1 ^ (val >> shift)) ==> bic
11691 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11692                          iRegL src1, iRegL src2,
11693                          immI src3, immL_M1 src4) %{
11694   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11695   ins_cost(1.9 * INSN_COST);
11696   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11697 
11698   ins_encode %{
11699     __ bic(as_Register($dst$$reg),
11700               as_Register($src1$$reg),
11701               as_Register($src2$$reg),
11702               Assembler::ASR,
11703               $src3$$constant & 0x3f);
11704   %}
11705 
11706   ins_pipe(ialu_reg_reg_shift);
11707 %}
11708 
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 // val & (-1 ^ (val ror shift)) ==> bicw
11712 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11713                          iRegIorL2I src1, iRegIorL2I src2,
11714                          immI src3, immI_M1 src4) %{
11715   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11716   ins_cost(1.9 * INSN_COST);
11717   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11718 
11719   ins_encode %{
11720     __ bicw(as_Register($dst$$reg),
11721               as_Register($src1$$reg),
11722               as_Register($src2$$reg),
11723               Assembler::ROR,
11724               $src3$$constant & 0x1f);
11725   %}
11726 
11727   ins_pipe(ialu_reg_reg_shift);
11728 %}
11729 
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732 // val & (-1 ^ (val ror shift)) ==> bic
11733 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11734                          iRegL src1, iRegL src2,
11735                          immI src3, immL_M1 src4) %{
11736   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11737   ins_cost(1.9 * INSN_COST);
11738   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11739 
11740   ins_encode %{
11741     __ bic(as_Register($dst$$reg),
11742               as_Register($src1$$reg),
11743               as_Register($src2$$reg),
11744               Assembler::ROR,
11745               $src3$$constant & 0x3f);
11746   %}
11747 
11748   ins_pipe(ialu_reg_reg_shift);
11749 %}
11750 
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 // val & (-1 ^ (val << shift)) ==> bicw
11754 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11755                          iRegIorL2I src1, iRegIorL2I src2,
11756                          immI src3, immI_M1 src4) %{
11757   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11758   ins_cost(1.9 * INSN_COST);
11759   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11760 
11761   ins_encode %{
11762     __ bicw(as_Register($dst$$reg),
11763               as_Register($src1$$reg),
11764               as_Register($src2$$reg),
11765               Assembler::LSL,
11766               $src3$$constant & 0x1f);
11767   %}
11768 
11769   ins_pipe(ialu_reg_reg_shift);
11770 %}
11771 
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 // val & (-1 ^ (val << shift)) ==> bic
11775 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11776                          iRegL src1, iRegL src2,
11777                          immI src3, immL_M1 src4) %{
11778   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11779   ins_cost(1.9 * INSN_COST);
11780   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11781 
11782   ins_encode %{
11783     __ bic(as_Register($dst$$reg),
11784               as_Register($src1$$reg),
11785               as_Register($src2$$reg),
11786               Assembler::LSL,
11787               $src3$$constant & 0x3f);
11788   %}
11789 
11790   ins_pipe(ialu_reg_reg_shift);
11791 %}
11792 
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11796 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11797                          iRegIorL2I src1, iRegIorL2I src2,
11798                          immI src3, immI_M1 src4) %{
11799   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11800   ins_cost(1.9 * INSN_COST);
11801   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11802 
11803   ins_encode %{
11804     __ eonw(as_Register($dst$$reg),
11805               as_Register($src1$$reg),
11806               as_Register($src2$$reg),
11807               Assembler::LSR,
11808               $src3$$constant & 0x1f);
11809   %}
11810 
11811   ins_pipe(ialu_reg_reg_shift);
11812 %}
11813 
11814 // This pattern is automatically generated from aarch64_ad.m4
11815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11816 // val ^ (-1 ^ (val >>> shift)) ==> eon
11817 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11818                          iRegL src1, iRegL src2,
11819                          immI src3, immL_M1 src4) %{
11820   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11821   ins_cost(1.9 * INSN_COST);
11822   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11823 
11824   ins_encode %{
11825     __ eon(as_Register($dst$$reg),
11826               as_Register($src1$$reg),
11827               as_Register($src2$$reg),
11828               Assembler::LSR,
11829               $src3$$constant & 0x3f);
11830   %}
11831 
11832   ins_pipe(ialu_reg_reg_shift);
11833 %}
11834 
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 // val ^ (-1 ^ (val >> shift)) ==> eonw
11838 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11839                          iRegIorL2I src1, iRegIorL2I src2,
11840                          immI src3, immI_M1 src4) %{
11841   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11842   ins_cost(1.9 * INSN_COST);
11843   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11844 
11845   ins_encode %{
11846     __ eonw(as_Register($dst$$reg),
11847               as_Register($src1$$reg),
11848               as_Register($src2$$reg),
11849               Assembler::ASR,
11850               $src3$$constant & 0x1f);
11851   %}
11852 
11853   ins_pipe(ialu_reg_reg_shift);
11854 %}
11855 
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 // val ^ (-1 ^ (val >> shift)) ==> eon
11859 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11860                          iRegL src1, iRegL src2,
11861                          immI src3, immL_M1 src4) %{
11862   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11863   ins_cost(1.9 * INSN_COST);
11864   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11865 
11866   ins_encode %{
11867     __ eon(as_Register($dst$$reg),
11868               as_Register($src1$$reg),
11869               as_Register($src2$$reg),
11870               Assembler::ASR,
11871               $src3$$constant & 0x3f);
11872   %}
11873 
11874   ins_pipe(ialu_reg_reg_shift);
11875 %}
11876 
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 // val ^ (-1 ^ (val ror shift)) ==> eonw
11880 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11881                          iRegIorL2I src1, iRegIorL2I src2,
11882                          immI src3, immI_M1 src4) %{
11883   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11884   ins_cost(1.9 * INSN_COST);
11885   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11886 
11887   ins_encode %{
11888     __ eonw(as_Register($dst$$reg),
11889               as_Register($src1$$reg),
11890               as_Register($src2$$reg),
11891               Assembler::ROR,
11892               $src3$$constant & 0x1f);
11893   %}
11894 
11895   ins_pipe(ialu_reg_reg_shift);
11896 %}
11897 
11898 // This pattern is automatically generated from aarch64_ad.m4
11899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11900 // val ^ (-1 ^ (val ror shift)) ==> eon
11901 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11902                          iRegL src1, iRegL src2,
11903                          immI src3, immL_M1 src4) %{
11904   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11905   ins_cost(1.9 * INSN_COST);
11906   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11907 
11908   ins_encode %{
11909     __ eon(as_Register($dst$$reg),
11910               as_Register($src1$$reg),
11911               as_Register($src2$$reg),
11912               Assembler::ROR,
11913               $src3$$constant & 0x3f);
11914   %}
11915 
11916   ins_pipe(ialu_reg_reg_shift);
11917 %}
11918 
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 // val ^ (-1 ^ (val << shift)) ==> eonw
11922 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11923                          iRegIorL2I src1, iRegIorL2I src2,
11924                          immI src3, immI_M1 src4) %{
11925   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11926   ins_cost(1.9 * INSN_COST);
11927   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11928 
11929   ins_encode %{
11930     __ eonw(as_Register($dst$$reg),
11931               as_Register($src1$$reg),
11932               as_Register($src2$$reg),
11933               Assembler::LSL,
11934               $src3$$constant & 0x1f);
11935   %}
11936 
11937   ins_pipe(ialu_reg_reg_shift);
11938 %}
11939 
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 // val ^ (-1 ^ (val << shift)) ==> eon
11943 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11944                          iRegL src1, iRegL src2,
11945                          immI src3, immL_M1 src4) %{
11946   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11947   ins_cost(1.9 * INSN_COST);
11948   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11949 
11950   ins_encode %{
11951     __ eon(as_Register($dst$$reg),
11952               as_Register($src1$$reg),
11953               as_Register($src2$$reg),
11954               Assembler::LSL,
11955               $src3$$constant & 0x3f);
11956   %}
11957 
11958   ins_pipe(ialu_reg_reg_shift);
11959 %}
11960 
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 // val | (-1 ^ (val >>> shift)) ==> ornw
11964 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11965                          iRegIorL2I src1, iRegIorL2I src2,
11966                          immI src3, immI_M1 src4) %{
11967   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11968   ins_cost(1.9 * INSN_COST);
11969   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11970 
11971   ins_encode %{
11972     __ ornw(as_Register($dst$$reg),
11973               as_Register($src1$$reg),
11974               as_Register($src2$$reg),
11975               Assembler::LSR,
11976               $src3$$constant & 0x1f);
11977   %}
11978 
11979   ins_pipe(ialu_reg_reg_shift);
11980 %}
11981 
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 // val | (-1 ^ (val >>> shift)) ==> orn
11985 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11986                          iRegL src1, iRegL src2,
11987                          immI src3, immL_M1 src4) %{
11988   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11989   ins_cost(1.9 * INSN_COST);
11990   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11991 
11992   ins_encode %{
11993     __ orn(as_Register($dst$$reg),
11994               as_Register($src1$$reg),
11995               as_Register($src2$$reg),
11996               Assembler::LSR,
11997               $src3$$constant & 0x3f);
11998   %}
11999 
12000   ins_pipe(ialu_reg_reg_shift);
12001 %}
12002 
12003 // This pattern is automatically generated from aarch64_ad.m4
12004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12005 // val | (-1 ^ (val >> shift)) ==> ornw
12006 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12007                          iRegIorL2I src1, iRegIorL2I src2,
12008                          immI src3, immI_M1 src4) %{
12009   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12010   ins_cost(1.9 * INSN_COST);
12011   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12012 
12013   ins_encode %{
12014     __ ornw(as_Register($dst$$reg),
12015               as_Register($src1$$reg),
12016               as_Register($src2$$reg),
12017               Assembler::ASR,
12018               $src3$$constant & 0x1f);
12019   %}
12020 
12021   ins_pipe(ialu_reg_reg_shift);
12022 %}
12023 
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 // val | (-1 ^ (val >> shift)) ==> orn
12027 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12028                          iRegL src1, iRegL src2,
12029                          immI src3, immL_M1 src4) %{
12030   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12031   ins_cost(1.9 * INSN_COST);
12032   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12033 
12034   ins_encode %{
12035     __ orn(as_Register($dst$$reg),
12036               as_Register($src1$$reg),
12037               as_Register($src2$$reg),
12038               Assembler::ASR,
12039               $src3$$constant & 0x3f);
12040   %}
12041 
12042   ins_pipe(ialu_reg_reg_shift);
12043 %}
12044 
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 // val | (-1 ^ (val ror shift)) ==> ornw
12048 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12049                          iRegIorL2I src1, iRegIorL2I src2,
12050                          immI src3, immI_M1 src4) %{
12051   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12052   ins_cost(1.9 * INSN_COST);
12053   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12054 
12055   ins_encode %{
12056     __ ornw(as_Register($dst$$reg),
12057               as_Register($src1$$reg),
12058               as_Register($src2$$reg),
12059               Assembler::ROR,
12060               $src3$$constant & 0x1f);
12061   %}
12062 
12063   ins_pipe(ialu_reg_reg_shift);
12064 %}
12065 
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 // val | (-1 ^ (val ror shift)) ==> orn
12069 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12070                          iRegL src1, iRegL src2,
12071                          immI src3, immL_M1 src4) %{
12072   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12073   ins_cost(1.9 * INSN_COST);
12074   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12075 
12076   ins_encode %{
12077     __ orn(as_Register($dst$$reg),
12078               as_Register($src1$$reg),
12079               as_Register($src2$$reg),
12080               Assembler::ROR,
12081               $src3$$constant & 0x3f);
12082   %}
12083 
12084   ins_pipe(ialu_reg_reg_shift);
12085 %}
12086 
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 // val | (-1 ^ (val << shift)) ==> ornw
12090 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12091                          iRegIorL2I src1, iRegIorL2I src2,
12092                          immI src3, immI_M1 src4) %{
12093   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12094   ins_cost(1.9 * INSN_COST);
12095   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12096 
12097   ins_encode %{
12098     __ ornw(as_Register($dst$$reg),
12099               as_Register($src1$$reg),
12100               as_Register($src2$$reg),
12101               Assembler::LSL,
12102               $src3$$constant & 0x1f);
12103   %}
12104 
12105   ins_pipe(ialu_reg_reg_shift);
12106 %}
12107 
12108 // This pattern is automatically generated from aarch64_ad.m4
12109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12110 // val | (-1 ^ (val << shift)) ==> orn
12111 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12112                          iRegL src1, iRegL src2,
12113                          immI src3, immL_M1 src4) %{
12114   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12115   ins_cost(1.9 * INSN_COST);
12116   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12117 
12118   ins_encode %{
12119     __ orn(as_Register($dst$$reg),
12120               as_Register($src1$$reg),
12121               as_Register($src2$$reg),
12122               Assembler::LSL,
12123               $src3$$constant & 0x3f);
12124   %}
12125 
12126   ins_pipe(ialu_reg_reg_shift);
12127 %}
12128 
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12132                          iRegIorL2I src1, iRegIorL2I src2,
12133                          immI src3) %{
12134   match(Set dst (AndI src1 (URShiftI src2 src3)));
12135 
12136   ins_cost(1.9 * INSN_COST);
12137   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12138 
12139   ins_encode %{
12140     __ andw(as_Register($dst$$reg),
12141               as_Register($src1$$reg),
12142               as_Register($src2$$reg),
12143               Assembler::LSR,
12144               $src3$$constant & 0x1f);
12145   %}
12146 
12147   ins_pipe(ialu_reg_reg_shift);
12148 %}
12149 
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12153                          iRegL src1, iRegL src2,
12154                          immI src3) %{
12155   match(Set dst (AndL src1 (URShiftL src2 src3)));
12156 
12157   ins_cost(1.9 * INSN_COST);
12158   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12159 
12160   ins_encode %{
12161     __ andr(as_Register($dst$$reg),
12162               as_Register($src1$$reg),
12163               as_Register($src2$$reg),
12164               Assembler::LSR,
12165               $src3$$constant & 0x3f);
12166   %}
12167 
12168   ins_pipe(ialu_reg_reg_shift);
12169 %}
12170 
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12174                          iRegIorL2I src1, iRegIorL2I src2,
12175                          immI src3) %{
12176   match(Set dst (AndI src1 (RShiftI src2 src3)));
12177 
12178   ins_cost(1.9 * INSN_COST);
12179   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12180 
12181   ins_encode %{
12182     __ andw(as_Register($dst$$reg),
12183               as_Register($src1$$reg),
12184               as_Register($src2$$reg),
12185               Assembler::ASR,
12186               $src3$$constant & 0x1f);
12187   %}
12188 
12189   ins_pipe(ialu_reg_reg_shift);
12190 %}
12191 
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12195                          iRegL src1, iRegL src2,
12196                          immI src3) %{
12197   match(Set dst (AndL src1 (RShiftL src2 src3)));
12198 
12199   ins_cost(1.9 * INSN_COST);
12200   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12201 
12202   ins_encode %{
12203     __ andr(as_Register($dst$$reg),
12204               as_Register($src1$$reg),
12205               as_Register($src2$$reg),
12206               Assembler::ASR,
12207               $src3$$constant & 0x3f);
12208   %}
12209 
12210   ins_pipe(ialu_reg_reg_shift);
12211 %}
12212 
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12216                          iRegIorL2I src1, iRegIorL2I src2,
12217                          immI src3) %{
12218   match(Set dst (AndI src1 (LShiftI src2 src3)));
12219 
12220   ins_cost(1.9 * INSN_COST);
12221   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12222 
12223   ins_encode %{
12224     __ andw(as_Register($dst$$reg),
12225               as_Register($src1$$reg),
12226               as_Register($src2$$reg),
12227               Assembler::LSL,
12228               $src3$$constant & 0x1f);
12229   %}
12230 
12231   ins_pipe(ialu_reg_reg_shift);
12232 %}
12233 
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12237                          iRegL src1, iRegL src2,
12238                          immI src3) %{
12239   match(Set dst (AndL src1 (LShiftL src2 src3)));
12240 
12241   ins_cost(1.9 * INSN_COST);
12242   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12243 
12244   ins_encode %{
12245     __ andr(as_Register($dst$$reg),
12246               as_Register($src1$$reg),
12247               as_Register($src2$$reg),
12248               Assembler::LSL,
12249               $src3$$constant & 0x3f);
12250   %}
12251 
12252   ins_pipe(ialu_reg_reg_shift);
12253 %}
12254 
12255 // This pattern is automatically generated from aarch64_ad.m4
12256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12257 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12258                          iRegIorL2I src1, iRegIorL2I src2,
12259                          immI src3) %{
12260   match(Set dst (AndI src1 (RotateRight src2 src3)));
12261 
12262   ins_cost(1.9 * INSN_COST);
12263   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12264 
12265   ins_encode %{
12266     __ andw(as_Register($dst$$reg),
12267               as_Register($src1$$reg),
12268               as_Register($src2$$reg),
12269               Assembler::ROR,
12270               $src3$$constant & 0x1f);
12271   %}
12272 
12273   ins_pipe(ialu_reg_reg_shift);
12274 %}
12275 
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12279                          iRegL src1, iRegL src2,
12280                          immI src3) %{
12281   match(Set dst (AndL src1 (RotateRight src2 src3)));
12282 
12283   ins_cost(1.9 * INSN_COST);
12284   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12285 
12286   ins_encode %{
12287     __ andr(as_Register($dst$$reg),
12288               as_Register($src1$$reg),
12289               as_Register($src2$$reg),
12290               Assembler::ROR,
12291               $src3$$constant & 0x3f);
12292   %}
12293 
12294   ins_pipe(ialu_reg_reg_shift);
12295 %}
12296 
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12300                          iRegIorL2I src1, iRegIorL2I src2,
12301                          immI src3) %{
12302   match(Set dst (XorI src1 (URShiftI src2 src3)));
12303 
12304   ins_cost(1.9 * INSN_COST);
12305   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12306 
12307   ins_encode %{
12308     __ eorw(as_Register($dst$$reg),
12309               as_Register($src1$$reg),
12310               as_Register($src2$$reg),
12311               Assembler::LSR,
12312               $src3$$constant & 0x1f);
12313   %}
12314 
12315   ins_pipe(ialu_reg_reg_shift);
12316 %}
12317 
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12321                          iRegL src1, iRegL src2,
12322                          immI src3) %{
12323   match(Set dst (XorL src1 (URShiftL src2 src3)));
12324 
12325   ins_cost(1.9 * INSN_COST);
12326   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12327 
12328   ins_encode %{
12329     __ eor(as_Register($dst$$reg),
12330               as_Register($src1$$reg),
12331               as_Register($src2$$reg),
12332               Assembler::LSR,
12333               $src3$$constant & 0x3f);
12334   %}
12335 
12336   ins_pipe(ialu_reg_reg_shift);
12337 %}
12338 
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12342                          iRegIorL2I src1, iRegIorL2I src2,
12343                          immI src3) %{
12344   match(Set dst (XorI src1 (RShiftI src2 src3)));
12345 
12346   ins_cost(1.9 * INSN_COST);
12347   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12348 
12349   ins_encode %{
12350     __ eorw(as_Register($dst$$reg),
12351               as_Register($src1$$reg),
12352               as_Register($src2$$reg),
12353               Assembler::ASR,
12354               $src3$$constant & 0x1f);
12355   %}
12356 
12357   ins_pipe(ialu_reg_reg_shift);
12358 %}
12359 
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12363                          iRegL src1, iRegL src2,
12364                          immI src3) %{
12365   match(Set dst (XorL src1 (RShiftL src2 src3)));
12366 
12367   ins_cost(1.9 * INSN_COST);
12368   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12369 
12370   ins_encode %{
12371     __ eor(as_Register($dst$$reg),
12372               as_Register($src1$$reg),
12373               as_Register($src2$$reg),
12374               Assembler::ASR,
12375               $src3$$constant & 0x3f);
12376   %}
12377 
12378   ins_pipe(ialu_reg_reg_shift);
12379 %}
12380 
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12384                          iRegIorL2I src1, iRegIorL2I src2,
12385                          immI src3) %{
12386   match(Set dst (XorI src1 (LShiftI src2 src3)));
12387 
12388   ins_cost(1.9 * INSN_COST);
12389   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12390 
12391   ins_encode %{
12392     __ eorw(as_Register($dst$$reg),
12393               as_Register($src1$$reg),
12394               as_Register($src2$$reg),
12395               Assembler::LSL,
12396               $src3$$constant & 0x1f);
12397   %}
12398 
12399   ins_pipe(ialu_reg_reg_shift);
12400 %}
12401 
12402 // This pattern is automatically generated from aarch64_ad.m4
12403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12404 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12405                          iRegL src1, iRegL src2,
12406                          immI src3) %{
12407   match(Set dst (XorL src1 (LShiftL src2 src3)));
12408 
12409   ins_cost(1.9 * INSN_COST);
12410   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12411 
12412   ins_encode %{
12413     __ eor(as_Register($dst$$reg),
12414               as_Register($src1$$reg),
12415               as_Register($src2$$reg),
12416               Assembler::LSL,
12417               $src3$$constant & 0x3f);
12418   %}
12419 
12420   ins_pipe(ialu_reg_reg_shift);
12421 %}
12422 
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12426                          iRegIorL2I src1, iRegIorL2I src2,
12427                          immI src3) %{
12428   match(Set dst (XorI src1 (RotateRight src2 src3)));
12429 
12430   ins_cost(1.9 * INSN_COST);
12431   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12432 
12433   ins_encode %{
12434     __ eorw(as_Register($dst$$reg),
12435               as_Register($src1$$reg),
12436               as_Register($src2$$reg),
12437               Assembler::ROR,
12438               $src3$$constant & 0x1f);
12439   %}
12440 
12441   ins_pipe(ialu_reg_reg_shift);
12442 %}
12443 
12444 // This pattern is automatically generated from aarch64_ad.m4
12445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12446 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12447                          iRegL src1, iRegL src2,
12448                          immI src3) %{
12449   match(Set dst (XorL src1 (RotateRight src2 src3)));
12450 
12451   ins_cost(1.9 * INSN_COST);
12452   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12453 
12454   ins_encode %{
12455     __ eor(as_Register($dst$$reg),
12456               as_Register($src1$$reg),
12457               as_Register($src2$$reg),
12458               Assembler::ROR,
12459               $src3$$constant & 0x3f);
12460   %}
12461 
12462   ins_pipe(ialu_reg_reg_shift);
12463 %}
12464 
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12468                          iRegIorL2I src1, iRegIorL2I src2,
12469                          immI src3) %{
12470   match(Set dst (OrI src1 (URShiftI src2 src3)));
12471 
12472   ins_cost(1.9 * INSN_COST);
12473   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12474 
12475   ins_encode %{
12476     __ orrw(as_Register($dst$$reg),
12477               as_Register($src1$$reg),
12478               as_Register($src2$$reg),
12479               Assembler::LSR,
12480               $src3$$constant & 0x1f);
12481   %}
12482 
12483   ins_pipe(ialu_reg_reg_shift);
12484 %}
12485 
12486 // This pattern is automatically generated from aarch64_ad.m4
12487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12488 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12489                          iRegL src1, iRegL src2,
12490                          immI src3) %{
12491   match(Set dst (OrL src1 (URShiftL src2 src3)));
12492 
12493   ins_cost(1.9 * INSN_COST);
12494   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12495 
12496   ins_encode %{
12497     __ orr(as_Register($dst$$reg),
12498               as_Register($src1$$reg),
12499               as_Register($src2$$reg),
12500               Assembler::LSR,
12501               $src3$$constant & 0x3f);
12502   %}
12503 
12504   ins_pipe(ialu_reg_reg_shift);
12505 %}
12506 
12507 // This pattern is automatically generated from aarch64_ad.m4
12508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12509 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12510                          iRegIorL2I src1, iRegIorL2I src2,
12511                          immI src3) %{
12512   match(Set dst (OrI src1 (RShiftI src2 src3)));
12513 
12514   ins_cost(1.9 * INSN_COST);
12515   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12516 
12517   ins_encode %{
12518     __ orrw(as_Register($dst$$reg),
12519               as_Register($src1$$reg),
12520               as_Register($src2$$reg),
12521               Assembler::ASR,
12522               $src3$$constant & 0x1f);
12523   %}
12524 
12525   ins_pipe(ialu_reg_reg_shift);
12526 %}
12527 
12528 // This pattern is automatically generated from aarch64_ad.m4
12529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12530 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12531                          iRegL src1, iRegL src2,
12532                          immI src3) %{
12533   match(Set dst (OrL src1 (RShiftL src2 src3)));
12534 
12535   ins_cost(1.9 * INSN_COST);
12536   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12537 
12538   ins_encode %{
12539     __ orr(as_Register($dst$$reg),
12540               as_Register($src1$$reg),
12541               as_Register($src2$$reg),
12542               Assembler::ASR,
12543               $src3$$constant & 0x3f);
12544   %}
12545 
12546   ins_pipe(ialu_reg_reg_shift);
12547 %}
12548 
12549 // This pattern is automatically generated from aarch64_ad.m4
12550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12551 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12552                          iRegIorL2I src1, iRegIorL2I src2,
12553                          immI src3) %{
12554   match(Set dst (OrI src1 (LShiftI src2 src3)));
12555 
12556   ins_cost(1.9 * INSN_COST);
12557   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12558 
12559   ins_encode %{
12560     __ orrw(as_Register($dst$$reg),
12561               as_Register($src1$$reg),
12562               as_Register($src2$$reg),
12563               Assembler::LSL,
12564               $src3$$constant & 0x1f);
12565   %}
12566 
12567   ins_pipe(ialu_reg_reg_shift);
12568 %}
12569 
12570 // This pattern is automatically generated from aarch64_ad.m4
12571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12572 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12573                          iRegL src1, iRegL src2,
12574                          immI src3) %{
12575   match(Set dst (OrL src1 (LShiftL src2 src3)));
12576 
12577   ins_cost(1.9 * INSN_COST);
12578   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12579 
12580   ins_encode %{
12581     __ orr(as_Register($dst$$reg),
12582               as_Register($src1$$reg),
12583               as_Register($src2$$reg),
12584               Assembler::LSL,
12585               $src3$$constant & 0x3f);
12586   %}
12587 
12588   ins_pipe(ialu_reg_reg_shift);
12589 %}
12590 
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12594                          iRegIorL2I src1, iRegIorL2I src2,
12595                          immI src3) %{
12596   match(Set dst (OrI src1 (RotateRight src2 src3)));
12597 
12598   ins_cost(1.9 * INSN_COST);
12599   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12600 
12601   ins_encode %{
12602     __ orrw(as_Register($dst$$reg),
12603               as_Register($src1$$reg),
12604               as_Register($src2$$reg),
12605               Assembler::ROR,
12606               $src3$$constant & 0x1f);
12607   %}
12608 
12609   ins_pipe(ialu_reg_reg_shift);
12610 %}
12611 
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12615                          iRegL src1, iRegL src2,
12616                          immI src3) %{
12617   match(Set dst (OrL src1 (RotateRight src2 src3)));
12618 
12619   ins_cost(1.9 * INSN_COST);
12620   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12621 
12622   ins_encode %{
12623     __ orr(as_Register($dst$$reg),
12624               as_Register($src1$$reg),
12625               as_Register($src2$$reg),
12626               Assembler::ROR,
12627               $src3$$constant & 0x3f);
12628   %}
12629 
12630   ins_pipe(ialu_reg_reg_shift);
12631 %}
12632 
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12636                          iRegIorL2I src1, iRegIorL2I src2,
12637                          immI src3) %{
12638   match(Set dst (AddI src1 (URShiftI src2 src3)));
12639 
12640   ins_cost(1.9 * INSN_COST);
12641   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12642 
12643   ins_encode %{
12644     __ addw(as_Register($dst$$reg),
12645               as_Register($src1$$reg),
12646               as_Register($src2$$reg),
12647               Assembler::LSR,
12648               $src3$$constant & 0x1f);
12649   %}
12650 
12651   ins_pipe(ialu_reg_reg_shift);
12652 %}
12653 
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12657                          iRegL src1, iRegL src2,
12658                          immI src3) %{
12659   match(Set dst (AddL src1 (URShiftL src2 src3)));
12660 
12661   ins_cost(1.9 * INSN_COST);
12662   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12663 
12664   ins_encode %{
12665     __ add(as_Register($dst$$reg),
12666               as_Register($src1$$reg),
12667               as_Register($src2$$reg),
12668               Assembler::LSR,
12669               $src3$$constant & 0x3f);
12670   %}
12671 
12672   ins_pipe(ialu_reg_reg_shift);
12673 %}
12674 
12675 // This pattern is automatically generated from aarch64_ad.m4
12676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12677 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12678                          iRegIorL2I src1, iRegIorL2I src2,
12679                          immI src3) %{
12680   match(Set dst (AddI src1 (RShiftI src2 src3)));
12681 
12682   ins_cost(1.9 * INSN_COST);
12683   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12684 
12685   ins_encode %{
12686     __ addw(as_Register($dst$$reg),
12687               as_Register($src1$$reg),
12688               as_Register($src2$$reg),
12689               Assembler::ASR,
12690               $src3$$constant & 0x1f);
12691   %}
12692 
12693   ins_pipe(ialu_reg_reg_shift);
12694 %}
12695 
12696 // This pattern is automatically generated from aarch64_ad.m4
12697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12698 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12699                          iRegL src1, iRegL src2,
12700                          immI src3) %{
12701   match(Set dst (AddL src1 (RShiftL src2 src3)));
12702 
12703   ins_cost(1.9 * INSN_COST);
12704   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12705 
12706   ins_encode %{
12707     __ add(as_Register($dst$$reg),
12708               as_Register($src1$$reg),
12709               as_Register($src2$$reg),
12710               Assembler::ASR,
12711               $src3$$constant & 0x3f);
12712   %}
12713 
12714   ins_pipe(ialu_reg_reg_shift);
12715 %}
12716 
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12720                          iRegIorL2I src1, iRegIorL2I src2,
12721                          immI src3) %{
12722   match(Set dst (AddI src1 (LShiftI src2 src3)));
12723 
12724   ins_cost(1.9 * INSN_COST);
12725   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12726 
12727   ins_encode %{
12728     __ addw(as_Register($dst$$reg),
12729               as_Register($src1$$reg),
12730               as_Register($src2$$reg),
12731               Assembler::LSL,
12732               $src3$$constant & 0x1f);
12733   %}
12734 
12735   ins_pipe(ialu_reg_reg_shift);
12736 %}
12737 
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12741                          iRegL src1, iRegL src2,
12742                          immI src3) %{
12743   match(Set dst (AddL src1 (LShiftL src2 src3)));
12744 
12745   ins_cost(1.9 * INSN_COST);
12746   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12747 
12748   ins_encode %{
12749     __ add(as_Register($dst$$reg),
12750               as_Register($src1$$reg),
12751               as_Register($src2$$reg),
12752               Assembler::LSL,
12753               $src3$$constant & 0x3f);
12754   %}
12755 
12756   ins_pipe(ialu_reg_reg_shift);
12757 %}
12758 
12759 // This pattern is automatically generated from aarch64_ad.m4
12760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12761 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12762                          iRegIorL2I src1, iRegIorL2I src2,
12763                          immI src3) %{
12764   match(Set dst (SubI src1 (URShiftI src2 src3)));
12765 
12766   ins_cost(1.9 * INSN_COST);
12767   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12768 
12769   ins_encode %{
12770     __ subw(as_Register($dst$$reg),
12771               as_Register($src1$$reg),
12772               as_Register($src2$$reg),
12773               Assembler::LSR,
12774               $src3$$constant & 0x1f);
12775   %}
12776 
12777   ins_pipe(ialu_reg_reg_shift);
12778 %}
12779 
12780 // This pattern is automatically generated from aarch64_ad.m4
12781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12782 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12783                          iRegL src1, iRegL src2,
12784                          immI src3) %{
12785   match(Set dst (SubL src1 (URShiftL src2 src3)));
12786 
12787   ins_cost(1.9 * INSN_COST);
12788   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12789 
12790   ins_encode %{
12791     __ sub(as_Register($dst$$reg),
12792               as_Register($src1$$reg),
12793               as_Register($src2$$reg),
12794               Assembler::LSR,
12795               $src3$$constant & 0x3f);
12796   %}
12797 
12798   ins_pipe(ialu_reg_reg_shift);
12799 %}
12800 
12801 // This pattern is automatically generated from aarch64_ad.m4
12802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12803 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12804                          iRegIorL2I src1, iRegIorL2I src2,
12805                          immI src3) %{
12806   match(Set dst (SubI src1 (RShiftI src2 src3)));
12807 
12808   ins_cost(1.9 * INSN_COST);
12809   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12810 
12811   ins_encode %{
12812     __ subw(as_Register($dst$$reg),
12813               as_Register($src1$$reg),
12814               as_Register($src2$$reg),
12815               Assembler::ASR,
12816               $src3$$constant & 0x1f);
12817   %}
12818 
12819   ins_pipe(ialu_reg_reg_shift);
12820 %}
12821 
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12825                          iRegL src1, iRegL src2,
12826                          immI src3) %{
12827   match(Set dst (SubL src1 (RShiftL src2 src3)));
12828 
12829   ins_cost(1.9 * INSN_COST);
12830   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12831 
12832   ins_encode %{
12833     __ sub(as_Register($dst$$reg),
12834               as_Register($src1$$reg),
12835               as_Register($src2$$reg),
12836               Assembler::ASR,
12837               $src3$$constant & 0x3f);
12838   %}
12839 
12840   ins_pipe(ialu_reg_reg_shift);
12841 %}
12842 
12843 // This pattern is automatically generated from aarch64_ad.m4
12844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12845 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12846                          iRegIorL2I src1, iRegIorL2I src2,
12847                          immI src3) %{
12848   match(Set dst (SubI src1 (LShiftI src2 src3)));
12849 
12850   ins_cost(1.9 * INSN_COST);
12851   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12852 
12853   ins_encode %{
12854     __ subw(as_Register($dst$$reg),
12855               as_Register($src1$$reg),
12856               as_Register($src2$$reg),
12857               Assembler::LSL,
12858               $src3$$constant & 0x1f);
12859   %}
12860 
12861   ins_pipe(ialu_reg_reg_shift);
12862 %}
12863 
12864 // This pattern is automatically generated from aarch64_ad.m4
12865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12866 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12867                          iRegL src1, iRegL src2,
12868                          immI src3) %{
12869   match(Set dst (SubL src1 (LShiftL src2 src3)));
12870 
12871   ins_cost(1.9 * INSN_COST);
12872   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12873 
12874   ins_encode %{
12875     __ sub(as_Register($dst$$reg),
12876               as_Register($src1$$reg),
12877               as_Register($src2$$reg),
12878               Assembler::LSL,
12879               $src3$$constant & 0x3f);
12880   %}
12881 
12882   ins_pipe(ialu_reg_reg_shift);
12883 %}
12884 
12885 // This pattern is automatically generated from aarch64_ad.m4
12886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12887 
12888 // Shift Left followed by Shift Right.
12889 // This idiom is used by the compiler for the i2b bytecode etc.
12890 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12891 %{
12892   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12893   ins_cost(INSN_COST * 2);
12894   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12895   ins_encode %{
12896     int lshift = $lshift_count$$constant & 63;
12897     int rshift = $rshift_count$$constant & 63;
12898     int s = 63 - lshift;
12899     int r = (rshift - lshift) & 63;
12900     __ sbfm(as_Register($dst$$reg),
12901             as_Register($src$$reg),
12902             r, s);
12903   %}
12904 
12905   ins_pipe(ialu_reg_shift);
12906 %}
12907 
12908 // This pattern is automatically generated from aarch64_ad.m4
12909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12910 
12911 // Shift Left followed by Shift Right.
12912 // This idiom is used by the compiler for the i2b bytecode etc.
12913 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12914 %{
12915   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12916   ins_cost(INSN_COST * 2);
12917   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12918   ins_encode %{
12919     int lshift = $lshift_count$$constant & 31;
12920     int rshift = $rshift_count$$constant & 31;
12921     int s = 31 - lshift;
12922     int r = (rshift - lshift) & 31;
12923     __ sbfmw(as_Register($dst$$reg),
12924             as_Register($src$$reg),
12925             r, s);
12926   %}
12927 
12928   ins_pipe(ialu_reg_shift);
12929 %}
12930 
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 
12934 // Shift Left followed by Shift Right.
12935 // This idiom is used by the compiler for the i2b bytecode etc.
12936 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12937 %{
12938   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12939   ins_cost(INSN_COST * 2);
12940   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12941   ins_encode %{
12942     int lshift = $lshift_count$$constant & 63;
12943     int rshift = $rshift_count$$constant & 63;
12944     int s = 63 - lshift;
12945     int r = (rshift - lshift) & 63;
12946     __ ubfm(as_Register($dst$$reg),
12947             as_Register($src$$reg),
12948             r, s);
12949   %}
12950 
12951   ins_pipe(ialu_reg_shift);
12952 %}
12953 
12954 // This pattern is automatically generated from aarch64_ad.m4
12955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12956 
12957 // Shift Left followed by Shift Right.
12958 // This idiom is used by the compiler for the i2b bytecode etc.
12959 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12960 %{
12961   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12962   ins_cost(INSN_COST * 2);
12963   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12964   ins_encode %{
12965     int lshift = $lshift_count$$constant & 31;
12966     int rshift = $rshift_count$$constant & 31;
12967     int s = 31 - lshift;
12968     int r = (rshift - lshift) & 31;
12969     __ ubfmw(as_Register($dst$$reg),
12970             as_Register($src$$reg),
12971             r, s);
12972   %}
12973 
12974   ins_pipe(ialu_reg_shift);
12975 %}
12976 
12977 // Bitfield extract with shift & mask
12978 
12979 // This pattern is automatically generated from aarch64_ad.m4
12980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12981 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12982 %{
12983   match(Set dst (AndI (URShiftI src rshift) mask));
12984   // Make sure we are not going to exceed what ubfxw can do.
12985   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12986 
12987   ins_cost(INSN_COST);
12988   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12989   ins_encode %{
12990     int rshift = $rshift$$constant & 31;
12991     intptr_t mask = $mask$$constant;
12992     int width = exact_log2(mask+1);
12993     __ ubfxw(as_Register($dst$$reg),
12994             as_Register($src$$reg), rshift, width);
12995   %}
12996   ins_pipe(ialu_reg_shift);
12997 %}
12998 
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13002 %{
13003   match(Set dst (AndL (URShiftL src rshift) mask));
13004   // Make sure we are not going to exceed what ubfx can do.
13005   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13006 
13007   ins_cost(INSN_COST);
13008   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13009   ins_encode %{
13010     int rshift = $rshift$$constant & 63;
13011     intptr_t mask = $mask$$constant;
13012     int width = exact_log2_long(mask+1);
13013     __ ubfx(as_Register($dst$$reg),
13014             as_Register($src$$reg), rshift, width);
13015   %}
13016   ins_pipe(ialu_reg_shift);
13017 %}
13018 
13019 
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 
13023 // We can use ubfx when extending an And with a mask when we know mask
13024 // is positive.  We know that because immI_bitmask guarantees it.
13025 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13026 %{
13027   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13028   // Make sure we are not going to exceed what ubfxw can do.
13029   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13030 
13031   ins_cost(INSN_COST * 2);
13032   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13033   ins_encode %{
13034     int rshift = $rshift$$constant & 31;
13035     intptr_t mask = $mask$$constant;
13036     int width = exact_log2(mask+1);
13037     __ ubfx(as_Register($dst$$reg),
13038             as_Register($src$$reg), rshift, width);
13039   %}
13040   ins_pipe(ialu_reg_shift);
13041 %}
13042 
13043 
13044 // This pattern is automatically generated from aarch64_ad.m4
13045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13046 
13047 // We can use ubfiz when masking by a positive number and then left shifting the result.
13048 // We know that the mask is positive because immI_bitmask guarantees it.
13049 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13050 %{
13051   match(Set dst (LShiftI (AndI src mask) lshift));
13052   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13053 
13054   ins_cost(INSN_COST);
13055   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13056   ins_encode %{
13057     int lshift = $lshift$$constant & 31;
13058     intptr_t mask = $mask$$constant;
13059     int width = exact_log2(mask+1);
13060     __ ubfizw(as_Register($dst$$reg),
13061           as_Register($src$$reg), lshift, width);
13062   %}
13063   ins_pipe(ialu_reg_shift);
13064 %}
13065 
13066 // This pattern is automatically generated from aarch64_ad.m4
13067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13068 
13069 // We can use ubfiz when masking by a positive number and then left shifting the result.
13070 // We know that the mask is positive because immL_bitmask guarantees it.
13071 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13072 %{
13073   match(Set dst (LShiftL (AndL src mask) lshift));
13074   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13075 
13076   ins_cost(INSN_COST);
13077   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13078   ins_encode %{
13079     int lshift = $lshift$$constant & 63;
13080     intptr_t mask = $mask$$constant;
13081     int width = exact_log2_long(mask+1);
13082     __ ubfiz(as_Register($dst$$reg),
13083           as_Register($src$$reg), lshift, width);
13084   %}
13085   ins_pipe(ialu_reg_shift);
13086 %}
13087 
13088 // This pattern is automatically generated from aarch64_ad.m4
13089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13090 
13091 // We can use ubfiz when masking by a positive number and then left shifting the result.
13092 // We know that the mask is positive because immI_bitmask guarantees it.
13093 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13094 %{
13095   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13096   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13097 
13098   ins_cost(INSN_COST);
13099   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13100   ins_encode %{
13101     int lshift = $lshift$$constant & 31;
13102     intptr_t mask = $mask$$constant;
13103     int width = exact_log2(mask+1);
13104     __ ubfizw(as_Register($dst$$reg),
13105           as_Register($src$$reg), lshift, width);
13106   %}
13107   ins_pipe(ialu_reg_shift);
13108 %}
13109 
13110 // This pattern is automatically generated from aarch64_ad.m4
13111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13112 
13113 // We can use ubfiz when masking by a positive number and then left shifting the result.
13114 // We know that the mask is positive because immL_bitmask guarantees it.
13115 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13116 %{
13117   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13118   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13119 
13120   ins_cost(INSN_COST);
13121   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13122   ins_encode %{
13123     int lshift = $lshift$$constant & 63;
13124     intptr_t mask = $mask$$constant;
13125     int width = exact_log2_long(mask+1);
13126     __ ubfiz(as_Register($dst$$reg),
13127           as_Register($src$$reg), lshift, width);
13128   %}
13129   ins_pipe(ialu_reg_shift);
13130 %}
13131 
13132 
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135 
13136 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13137 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13138 %{
13139   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13140   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13141 
13142   ins_cost(INSN_COST);
13143   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13144   ins_encode %{
13145     int lshift = $lshift$$constant & 63;
13146     intptr_t mask = $mask$$constant;
13147     int width = exact_log2(mask+1);
13148     __ ubfiz(as_Register($dst$$reg),
13149              as_Register($src$$reg), lshift, width);
13150   %}
13151   ins_pipe(ialu_reg_shift);
13152 %}
13153 
13154 // This pattern is automatically generated from aarch64_ad.m4
13155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13156 
13157 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13158 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13159 %{
13160   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13161   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13162 
13163   ins_cost(INSN_COST);
13164   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13165   ins_encode %{
13166     int lshift = $lshift$$constant & 31;
13167     intptr_t mask = $mask$$constant;
13168     int width = exact_log2(mask+1);
13169     __ ubfiz(as_Register($dst$$reg),
13170              as_Register($src$$reg), lshift, width);
13171   %}
13172   ins_pipe(ialu_reg_shift);
13173 %}
13174 
13175 // This pattern is automatically generated from aarch64_ad.m4
13176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13177 
13178 // Can skip int2long conversions after AND with small bitmask
13179 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13180 %{
13181   match(Set dst (ConvI2L (AndI src msk)));
13182   ins_cost(INSN_COST);
13183   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13184   ins_encode %{
13185     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13186   %}
13187   ins_pipe(ialu_reg_shift);
13188 %}
13189 
13190 
13191 // Rotations
13192 // This pattern is automatically generated from aarch64_ad.m4
13193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13194 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13195 %{
13196   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13197   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13198 
13199   ins_cost(INSN_COST);
13200   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13201 
13202   ins_encode %{
13203     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13204             $rshift$$constant & 63);
13205   %}
13206   ins_pipe(ialu_reg_reg_extr);
13207 %}
13208 
13209 
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13213 %{
13214   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13215   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13216 
13217   ins_cost(INSN_COST);
13218   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13219 
13220   ins_encode %{
13221     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13222             $rshift$$constant & 31);
13223   %}
13224   ins_pipe(ialu_reg_reg_extr);
13225 %}
13226 
13227 
13228 // This pattern is automatically generated from aarch64_ad.m4
13229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13230 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13231 %{
13232   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13233   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13234 
13235   ins_cost(INSN_COST);
13236   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13237 
13238   ins_encode %{
13239     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13240             $rshift$$constant & 63);
13241   %}
13242   ins_pipe(ialu_reg_reg_extr);
13243 %}
13244 
13245 
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13249 %{
13250   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13251   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13252 
13253   ins_cost(INSN_COST);
13254   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13255 
13256   ins_encode %{
13257     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13258             $rshift$$constant & 31);
13259   %}
13260   ins_pipe(ialu_reg_reg_extr);
13261 %}
13262 
13263 
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13267 %{
13268   match(Set dst (RotateRight src shift));
13269 
13270   ins_cost(INSN_COST);
13271   format %{ "ror    $dst, $src, $shift" %}
13272 
13273   ins_encode %{
13274      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13275                $shift$$constant & 0x1f);
13276   %}
13277   ins_pipe(ialu_reg_reg_vshift);
13278 %}
13279 
13280 // This pattern is automatically generated from aarch64_ad.m4
13281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13282 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13283 %{
13284   match(Set dst (RotateRight src shift));
13285 
13286   ins_cost(INSN_COST);
13287   format %{ "ror    $dst, $src, $shift" %}
13288 
13289   ins_encode %{
13290      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13291                $shift$$constant & 0x3f);
13292   %}
13293   ins_pipe(ialu_reg_reg_vshift);
13294 %}
13295 
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13299 %{
13300   match(Set dst (RotateRight src shift));
13301 
13302   ins_cost(INSN_COST);
13303   format %{ "ror    $dst, $src, $shift" %}
13304 
13305   ins_encode %{
13306      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13307   %}
13308   ins_pipe(ialu_reg_reg_vshift);
13309 %}
13310 
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13314 %{
13315   match(Set dst (RotateRight src shift));
13316 
13317   ins_cost(INSN_COST);
13318   format %{ "ror    $dst, $src, $shift" %}
13319 
13320   ins_encode %{
13321      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13322   %}
13323   ins_pipe(ialu_reg_reg_vshift);
13324 %}
13325 
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13329 %{
13330   match(Set dst (RotateLeft src shift));
13331 
13332   ins_cost(INSN_COST);
13333   format %{ "rol    $dst, $src, $shift" %}
13334 
13335   ins_encode %{
13336      __ subw(rscratch1, zr, as_Register($shift$$reg));
13337      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13338   %}
13339   ins_pipe(ialu_reg_reg_vshift);
13340 %}
13341 
13342 // This pattern is automatically generated from aarch64_ad.m4
13343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13344 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13345 %{
13346   match(Set dst (RotateLeft src shift));
13347 
13348   ins_cost(INSN_COST);
13349   format %{ "rol    $dst, $src, $shift" %}
13350 
13351   ins_encode %{
13352      __ subw(rscratch1, zr, as_Register($shift$$reg));
13353      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13354   %}
13355   ins_pipe(ialu_reg_reg_vshift);
13356 %}
13357 
13358 
13359 // Add/subtract (extended)
13360 
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13364 %{
13365   match(Set dst (AddL src1 (ConvI2L src2)));
13366   ins_cost(INSN_COST);
13367   format %{ "add  $dst, $src1, $src2, sxtw" %}
13368 
13369    ins_encode %{
13370      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13371             as_Register($src2$$reg), ext::sxtw);
13372    %}
13373   ins_pipe(ialu_reg_reg);
13374 %}
13375 
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13379 %{
13380   match(Set dst (SubL src1 (ConvI2L src2)));
13381   ins_cost(INSN_COST);
13382   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13383 
13384    ins_encode %{
13385      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13386             as_Register($src2$$reg), ext::sxtw);
13387    %}
13388   ins_pipe(ialu_reg_reg);
13389 %}
13390 
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13394 %{
13395   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13396   ins_cost(INSN_COST);
13397   format %{ "add  $dst, $src1, $src2, sxth" %}
13398 
13399    ins_encode %{
13400      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13401             as_Register($src2$$reg), ext::sxth);
13402    %}
13403   ins_pipe(ialu_reg_reg);
13404 %}
13405 
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13409 %{
13410   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13411   ins_cost(INSN_COST);
13412   format %{ "add  $dst, $src1, $src2, sxtb" %}
13413 
13414    ins_encode %{
13415      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13416             as_Register($src2$$reg), ext::sxtb);
13417    %}
13418   ins_pipe(ialu_reg_reg);
13419 %}
13420 
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13424 %{
13425   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13426   ins_cost(INSN_COST);
13427   format %{ "add  $dst, $src1, $src2, uxtb" %}
13428 
13429    ins_encode %{
13430      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13431             as_Register($src2$$reg), ext::uxtb);
13432    %}
13433   ins_pipe(ialu_reg_reg);
13434 %}
13435 
13436 // This pattern is automatically generated from aarch64_ad.m4
13437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13438 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13439 %{
13440   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13441   ins_cost(INSN_COST);
13442   format %{ "add  $dst, $src1, $src2, sxth" %}
13443 
13444    ins_encode %{
13445      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13446             as_Register($src2$$reg), ext::sxth);
13447    %}
13448   ins_pipe(ialu_reg_reg);
13449 %}
13450 
13451 // This pattern is automatically generated from aarch64_ad.m4
13452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13453 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13454 %{
13455   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13456   ins_cost(INSN_COST);
13457   format %{ "add  $dst, $src1, $src2, sxtw" %}
13458 
13459    ins_encode %{
13460      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13461             as_Register($src2$$reg), ext::sxtw);
13462    %}
13463   ins_pipe(ialu_reg_reg);
13464 %}
13465 
13466 // This pattern is automatically generated from aarch64_ad.m4
13467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13468 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13469 %{
13470   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13471   ins_cost(INSN_COST);
13472   format %{ "add  $dst, $src1, $src2, sxtb" %}
13473 
13474    ins_encode %{
13475      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13476             as_Register($src2$$reg), ext::sxtb);
13477    %}
13478   ins_pipe(ialu_reg_reg);
13479 %}
13480 
13481 // This pattern is automatically generated from aarch64_ad.m4
13482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13483 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13484 %{
13485   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13486   ins_cost(INSN_COST);
13487   format %{ "add  $dst, $src1, $src2, uxtb" %}
13488 
13489    ins_encode %{
13490      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13491             as_Register($src2$$reg), ext::uxtb);
13492    %}
13493   ins_pipe(ialu_reg_reg);
13494 %}
13495 
13496 // This pattern is automatically generated from aarch64_ad.m4
13497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13498 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13499 %{
13500   match(Set dst (AddI src1 (AndI src2 mask)));
13501   ins_cost(INSN_COST);
13502   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13503 
13504    ins_encode %{
13505      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13506             as_Register($src2$$reg), ext::uxtb);
13507    %}
13508   ins_pipe(ialu_reg_reg);
13509 %}
13510 
13511 // This pattern is automatically generated from aarch64_ad.m4
13512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13513 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13514 %{
13515   match(Set dst (AddI src1 (AndI src2 mask)));
13516   ins_cost(INSN_COST);
13517   format %{ "addw  $dst, $src1, $src2, uxth" %}
13518 
13519    ins_encode %{
13520      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13521             as_Register($src2$$reg), ext::uxth);
13522    %}
13523   ins_pipe(ialu_reg_reg);
13524 %}
13525 
13526 // This pattern is automatically generated from aarch64_ad.m4
13527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13528 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13529 %{
13530   match(Set dst (AddL src1 (AndL src2 mask)));
13531   ins_cost(INSN_COST);
13532   format %{ "add  $dst, $src1, $src2, uxtb" %}
13533 
13534    ins_encode %{
13535      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13536             as_Register($src2$$reg), ext::uxtb);
13537    %}
13538   ins_pipe(ialu_reg_reg);
13539 %}
13540 
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13544 %{
13545   match(Set dst (AddL src1 (AndL src2 mask)));
13546   ins_cost(INSN_COST);
13547   format %{ "add  $dst, $src1, $src2, uxth" %}
13548 
13549    ins_encode %{
13550      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13551             as_Register($src2$$reg), ext::uxth);
13552    %}
13553   ins_pipe(ialu_reg_reg);
13554 %}
13555 
13556 // This pattern is automatically generated from aarch64_ad.m4
13557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13558 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13559 %{
13560   match(Set dst (AddL src1 (AndL src2 mask)));
13561   ins_cost(INSN_COST);
13562   format %{ "add  $dst, $src1, $src2, uxtw" %}
13563 
13564    ins_encode %{
13565      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13566             as_Register($src2$$reg), ext::uxtw);
13567    %}
13568   ins_pipe(ialu_reg_reg);
13569 %}
13570 
13571 // This pattern is automatically generated from aarch64_ad.m4
13572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13573 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13574 %{
13575   match(Set dst (SubI src1 (AndI src2 mask)));
13576   ins_cost(INSN_COST);
13577   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13578 
13579    ins_encode %{
13580      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13581             as_Register($src2$$reg), ext::uxtb);
13582    %}
13583   ins_pipe(ialu_reg_reg);
13584 %}
13585 
13586 // This pattern is automatically generated from aarch64_ad.m4
13587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13588 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13589 %{
13590   match(Set dst (SubI src1 (AndI src2 mask)));
13591   ins_cost(INSN_COST);
13592   format %{ "subw  $dst, $src1, $src2, uxth" %}
13593 
13594    ins_encode %{
13595      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13596             as_Register($src2$$reg), ext::uxth);
13597    %}
13598   ins_pipe(ialu_reg_reg);
13599 %}
13600 
13601 // This pattern is automatically generated from aarch64_ad.m4
13602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13603 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13604 %{
13605   match(Set dst (SubL src1 (AndL src2 mask)));
13606   ins_cost(INSN_COST);
13607   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13608 
13609    ins_encode %{
13610      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13611             as_Register($src2$$reg), ext::uxtb);
13612    %}
13613   ins_pipe(ialu_reg_reg);
13614 %}
13615 
13616 // This pattern is automatically generated from aarch64_ad.m4
13617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13618 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13619 %{
13620   match(Set dst (SubL src1 (AndL src2 mask)));
13621   ins_cost(INSN_COST);
13622   format %{ "sub  $dst, $src1, $src2, uxth" %}
13623 
13624    ins_encode %{
13625      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13626             as_Register($src2$$reg), ext::uxth);
13627    %}
13628   ins_pipe(ialu_reg_reg);
13629 %}
13630 
13631 // This pattern is automatically generated from aarch64_ad.m4
13632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13633 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13634 %{
13635   match(Set dst (SubL src1 (AndL src2 mask)));
13636   ins_cost(INSN_COST);
13637   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13638 
13639    ins_encode %{
13640      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13641             as_Register($src2$$reg), ext::uxtw);
13642    %}
13643   ins_pipe(ialu_reg_reg);
13644 %}
13645 
13646 
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13650 %{
13651   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13652   ins_cost(1.9 * INSN_COST);
13653   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13654 
13655    ins_encode %{
13656      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13657             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13658    %}
13659   ins_pipe(ialu_reg_reg_shift);
13660 %}
13661 
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13665 %{
13666   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13667   ins_cost(1.9 * INSN_COST);
13668   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13669 
13670    ins_encode %{
13671      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13672             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13673    %}
13674   ins_pipe(ialu_reg_reg_shift);
13675 %}
13676 
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13680 %{
13681   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13682   ins_cost(1.9 * INSN_COST);
13683   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13684 
13685    ins_encode %{
13686      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13687             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13688    %}
13689   ins_pipe(ialu_reg_reg_shift);
13690 %}
13691 
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13695 %{
13696   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13697   ins_cost(1.9 * INSN_COST);
13698   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13699 
13700    ins_encode %{
13701      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13702             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13703    %}
13704   ins_pipe(ialu_reg_reg_shift);
13705 %}
13706 
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13710 %{
13711   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13712   ins_cost(1.9 * INSN_COST);
13713   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13714 
13715    ins_encode %{
13716      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13717             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13718    %}
13719   ins_pipe(ialu_reg_reg_shift);
13720 %}
13721 
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13725 %{
13726   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13727   ins_cost(1.9 * INSN_COST);
13728   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13729 
13730    ins_encode %{
13731      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13732             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13733    %}
13734   ins_pipe(ialu_reg_reg_shift);
13735 %}
13736 
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13740 %{
13741   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13742   ins_cost(1.9 * INSN_COST);
13743   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13744 
13745    ins_encode %{
13746      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13747             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13748    %}
13749   ins_pipe(ialu_reg_reg_shift);
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13755 %{
13756   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13757   ins_cost(1.9 * INSN_COST);
13758   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13759 
13760    ins_encode %{
13761      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13762             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13763    %}
13764   ins_pipe(ialu_reg_reg_shift);
13765 %}
13766 
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13770 %{
13771   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13772   ins_cost(1.9 * INSN_COST);
13773   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13774 
13775    ins_encode %{
13776      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13777             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13778    %}
13779   ins_pipe(ialu_reg_reg_shift);
13780 %}
13781 
13782 // This pattern is automatically generated from aarch64_ad.m4
13783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13784 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13785 %{
13786   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13787   ins_cost(1.9 * INSN_COST);
13788   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13789 
13790    ins_encode %{
13791      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13792             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13793    %}
13794   ins_pipe(ialu_reg_reg_shift);
13795 %}
13796 
13797 // This pattern is automatically generated from aarch64_ad.m4
13798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13799 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13800 %{
13801   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13802   ins_cost(1.9 * INSN_COST);
13803   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13804 
13805    ins_encode %{
13806      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13807             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13808    %}
13809   ins_pipe(ialu_reg_reg_shift);
13810 %}
13811 
13812 // This pattern is automatically generated from aarch64_ad.m4
13813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13814 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13815 %{
13816   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13817   ins_cost(1.9 * INSN_COST);
13818   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13819 
13820    ins_encode %{
13821      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13822             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13823    %}
13824   ins_pipe(ialu_reg_reg_shift);
13825 %}
13826 
13827 // This pattern is automatically generated from aarch64_ad.m4
13828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13829 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13830 %{
13831   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13832   ins_cost(1.9 * INSN_COST);
13833   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13834 
13835    ins_encode %{
13836      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13837             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13838    %}
13839   ins_pipe(ialu_reg_reg_shift);
13840 %}
13841 
13842 // This pattern is automatically generated from aarch64_ad.m4
13843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13844 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13845 %{
13846   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13847   ins_cost(1.9 * INSN_COST);
13848   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13849 
13850    ins_encode %{
13851      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13852             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13853    %}
13854   ins_pipe(ialu_reg_reg_shift);
13855 %}
13856 
13857 // This pattern is automatically generated from aarch64_ad.m4
13858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13859 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13860 %{
13861   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13862   ins_cost(1.9 * INSN_COST);
13863   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13864 
13865    ins_encode %{
13866      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13867             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13868    %}
13869   ins_pipe(ialu_reg_reg_shift);
13870 %}
13871 
13872 // This pattern is automatically generated from aarch64_ad.m4
13873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13874 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13875 %{
13876   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13877   ins_cost(1.9 * INSN_COST);
13878   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13879 
13880    ins_encode %{
13881      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13882             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13883    %}
13884   ins_pipe(ialu_reg_reg_shift);
13885 %}
13886 
13887 // This pattern is automatically generated from aarch64_ad.m4
13888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13889 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13890 %{
13891   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13892   ins_cost(1.9 * INSN_COST);
13893   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13894 
13895    ins_encode %{
13896      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13897             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13898    %}
13899   ins_pipe(ialu_reg_reg_shift);
13900 %}
13901 
13902 // This pattern is automatically generated from aarch64_ad.m4
13903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13904 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13905 %{
13906   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13907   ins_cost(1.9 * INSN_COST);
13908   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13909 
13910    ins_encode %{
13911      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13912             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13913    %}
13914   ins_pipe(ialu_reg_reg_shift);
13915 %}
13916 
13917 // This pattern is automatically generated from aarch64_ad.m4
13918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13919 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13920 %{
13921   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13922   ins_cost(1.9 * INSN_COST);
13923   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13924 
13925    ins_encode %{
13926      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13927             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13928    %}
13929   ins_pipe(ialu_reg_reg_shift);
13930 %}
13931 
13932 // This pattern is automatically generated from aarch64_ad.m4
13933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13934 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13935 %{
13936   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13937   ins_cost(1.9 * INSN_COST);
13938   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13939 
13940    ins_encode %{
13941      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13942             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13943    %}
13944   ins_pipe(ialu_reg_reg_shift);
13945 %}
13946 
13947 // This pattern is automatically generated from aarch64_ad.m4
13948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13949 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13950 %{
13951   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13952   ins_cost(1.9 * INSN_COST);
13953   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13954 
13955    ins_encode %{
13956      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13957             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13958    %}
13959   ins_pipe(ialu_reg_reg_shift);
13960 %}
13961 
13962 // This pattern is automatically generated from aarch64_ad.m4
13963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13964 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13965 %{
13966   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13967   ins_cost(1.9 * INSN_COST);
13968   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13969 
13970    ins_encode %{
13971      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13972             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13973    %}
13974   ins_pipe(ialu_reg_reg_shift);
13975 %}
13976 
13977 
13978 
13979 // END This section of the file is automatically generated. Do not edit --------------
13980 
13981 
13982 // ============================================================================
13983 // Floating Point Arithmetic Instructions
13984 
13985 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13986   match(Set dst (AddF src1 src2));
13987 
13988   ins_cost(INSN_COST * 5);
13989   format %{ "fadds   $dst, $src1, $src2" %}
13990 
13991   ins_encode %{
13992     __ fadds(as_FloatRegister($dst$$reg),
13993              as_FloatRegister($src1$$reg),
13994              as_FloatRegister($src2$$reg));
13995   %}
13996 
13997   ins_pipe(fp_dop_reg_reg_s);
13998 %}
13999 
14000 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14001   match(Set dst (AddD src1 src2));
14002 
14003   ins_cost(INSN_COST * 5);
14004   format %{ "faddd   $dst, $src1, $src2" %}
14005 
14006   ins_encode %{
14007     __ faddd(as_FloatRegister($dst$$reg),
14008              as_FloatRegister($src1$$reg),
14009              as_FloatRegister($src2$$reg));
14010   %}
14011 
14012   ins_pipe(fp_dop_reg_reg_d);
14013 %}
14014 
14015 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14016   match(Set dst (SubF src1 src2));
14017 
14018   ins_cost(INSN_COST * 5);
14019   format %{ "fsubs   $dst, $src1, $src2" %}
14020 
14021   ins_encode %{
14022     __ fsubs(as_FloatRegister($dst$$reg),
14023              as_FloatRegister($src1$$reg),
14024              as_FloatRegister($src2$$reg));
14025   %}
14026 
14027   ins_pipe(fp_dop_reg_reg_s);
14028 %}
14029 
14030 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14031   match(Set dst (SubD src1 src2));
14032 
14033   ins_cost(INSN_COST * 5);
14034   format %{ "fsubd   $dst, $src1, $src2" %}
14035 
14036   ins_encode %{
14037     __ fsubd(as_FloatRegister($dst$$reg),
14038              as_FloatRegister($src1$$reg),
14039              as_FloatRegister($src2$$reg));
14040   %}
14041 
14042   ins_pipe(fp_dop_reg_reg_d);
14043 %}
14044 
14045 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14046   match(Set dst (MulF src1 src2));
14047 
14048   ins_cost(INSN_COST * 6);
14049   format %{ "fmuls   $dst, $src1, $src2" %}
14050 
14051   ins_encode %{
14052     __ fmuls(as_FloatRegister($dst$$reg),
14053              as_FloatRegister($src1$$reg),
14054              as_FloatRegister($src2$$reg));
14055   %}
14056 
14057   ins_pipe(fp_dop_reg_reg_s);
14058 %}
14059 
14060 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14061   match(Set dst (MulD src1 src2));
14062 
14063   ins_cost(INSN_COST * 6);
14064   format %{ "fmuld   $dst, $src1, $src2" %}
14065 
14066   ins_encode %{
14067     __ fmuld(as_FloatRegister($dst$$reg),
14068              as_FloatRegister($src1$$reg),
14069              as_FloatRegister($src2$$reg));
14070   %}
14071 
14072   ins_pipe(fp_dop_reg_reg_d);
14073 %}
14074 
14075 // src1 * src2 + src3
14076 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14077   predicate(UseFMA);
14078   match(Set dst (FmaF src3 (Binary src1 src2)));
14079 
14080   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14081 
14082   ins_encode %{
14083     __ fmadds(as_FloatRegister($dst$$reg),
14084              as_FloatRegister($src1$$reg),
14085              as_FloatRegister($src2$$reg),
14086              as_FloatRegister($src3$$reg));
14087   %}
14088 
14089   ins_pipe(pipe_class_default);
14090 %}
14091 
14092 // src1 * src2 + src3
14093 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14094   predicate(UseFMA);
14095   match(Set dst (FmaD src3 (Binary src1 src2)));
14096 
14097   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14098 
14099   ins_encode %{
14100     __ fmaddd(as_FloatRegister($dst$$reg),
14101              as_FloatRegister($src1$$reg),
14102              as_FloatRegister($src2$$reg),
14103              as_FloatRegister($src3$$reg));
14104   %}
14105 
14106   ins_pipe(pipe_class_default);
14107 %}
14108 
14109 // -src1 * src2 + src3
14110 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14111   predicate(UseFMA);
14112   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14113   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14114 
14115   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14116 
14117   ins_encode %{
14118     __ fmsubs(as_FloatRegister($dst$$reg),
14119               as_FloatRegister($src1$$reg),
14120               as_FloatRegister($src2$$reg),
14121               as_FloatRegister($src3$$reg));
14122   %}
14123 
14124   ins_pipe(pipe_class_default);
14125 %}
14126 
14127 // -src1 * src2 + src3
14128 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14129   predicate(UseFMA);
14130   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14131   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14132 
14133   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14134 
14135   ins_encode %{
14136     __ fmsubd(as_FloatRegister($dst$$reg),
14137               as_FloatRegister($src1$$reg),
14138               as_FloatRegister($src2$$reg),
14139               as_FloatRegister($src3$$reg));
14140   %}
14141 
14142   ins_pipe(pipe_class_default);
14143 %}
14144 
14145 // -src1 * src2 - src3
14146 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14147   predicate(UseFMA);
14148   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14149   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14150 
14151   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14152 
14153   ins_encode %{
14154     __ fnmadds(as_FloatRegister($dst$$reg),
14155                as_FloatRegister($src1$$reg),
14156                as_FloatRegister($src2$$reg),
14157                as_FloatRegister($src3$$reg));
14158   %}
14159 
14160   ins_pipe(pipe_class_default);
14161 %}
14162 
14163 // -src1 * src2 - src3
14164 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14165   predicate(UseFMA);
14166   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14167   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14168 
14169   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14170 
14171   ins_encode %{
14172     __ fnmaddd(as_FloatRegister($dst$$reg),
14173                as_FloatRegister($src1$$reg),
14174                as_FloatRegister($src2$$reg),
14175                as_FloatRegister($src3$$reg));
14176   %}
14177 
14178   ins_pipe(pipe_class_default);
14179 %}
14180 
14181 // src1 * src2 - src3
14182 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14183   predicate(UseFMA);
14184   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14185 
14186   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14187 
14188   ins_encode %{
14189     __ fnmsubs(as_FloatRegister($dst$$reg),
14190                as_FloatRegister($src1$$reg),
14191                as_FloatRegister($src2$$reg),
14192                as_FloatRegister($src3$$reg));
14193   %}
14194 
14195   ins_pipe(pipe_class_default);
14196 %}
14197 
14198 // src1 * src2 - src3
14199 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14200   predicate(UseFMA);
14201   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14202 
14203   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14204 
14205   ins_encode %{
14206   // n.b. insn name should be fnmsubd
14207     __ fnmsub(as_FloatRegister($dst$$reg),
14208               as_FloatRegister($src1$$reg),
14209               as_FloatRegister($src2$$reg),
14210               as_FloatRegister($src3$$reg));
14211   %}
14212 
14213   ins_pipe(pipe_class_default);
14214 %}
14215 
14216 
14217 // Math.max(FF)F
14218 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14219   match(Set dst (MaxF src1 src2));
14220 
14221   format %{ "fmaxs   $dst, $src1, $src2" %}
14222   ins_encode %{
14223     __ fmaxs(as_FloatRegister($dst$$reg),
14224              as_FloatRegister($src1$$reg),
14225              as_FloatRegister($src2$$reg));
14226   %}
14227 
14228   ins_pipe(fp_dop_reg_reg_s);
14229 %}
14230 
14231 // Math.min(FF)F
14232 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14233   match(Set dst (MinF src1 src2));
14234 
14235   format %{ "fmins   $dst, $src1, $src2" %}
14236   ins_encode %{
14237     __ fmins(as_FloatRegister($dst$$reg),
14238              as_FloatRegister($src1$$reg),
14239              as_FloatRegister($src2$$reg));
14240   %}
14241 
14242   ins_pipe(fp_dop_reg_reg_s);
14243 %}
14244 
14245 // Math.max(DD)D
14246 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14247   match(Set dst (MaxD src1 src2));
14248 
14249   format %{ "fmaxd   $dst, $src1, $src2" %}
14250   ins_encode %{
14251     __ fmaxd(as_FloatRegister($dst$$reg),
14252              as_FloatRegister($src1$$reg),
14253              as_FloatRegister($src2$$reg));
14254   %}
14255 
14256   ins_pipe(fp_dop_reg_reg_d);
14257 %}
14258 
14259 // Math.min(DD)D
14260 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14261   match(Set dst (MinD src1 src2));
14262 
14263   format %{ "fmind   $dst, $src1, $src2" %}
14264   ins_encode %{
14265     __ fmind(as_FloatRegister($dst$$reg),
14266              as_FloatRegister($src1$$reg),
14267              as_FloatRegister($src2$$reg));
14268   %}
14269 
14270   ins_pipe(fp_dop_reg_reg_d);
14271 %}
14272 
14273 
14274 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14275   match(Set dst (DivF src1  src2));
14276 
14277   ins_cost(INSN_COST * 18);
14278   format %{ "fdivs   $dst, $src1, $src2" %}
14279 
14280   ins_encode %{
14281     __ fdivs(as_FloatRegister($dst$$reg),
14282              as_FloatRegister($src1$$reg),
14283              as_FloatRegister($src2$$reg));
14284   %}
14285 
14286   ins_pipe(fp_div_s);
14287 %}
14288 
14289 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14290   match(Set dst (DivD src1  src2));
14291 
14292   ins_cost(INSN_COST * 32);
14293   format %{ "fdivd   $dst, $src1, $src2" %}
14294 
14295   ins_encode %{
14296     __ fdivd(as_FloatRegister($dst$$reg),
14297              as_FloatRegister($src1$$reg),
14298              as_FloatRegister($src2$$reg));
14299   %}
14300 
14301   ins_pipe(fp_div_d);
14302 %}
14303 
14304 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14305   match(Set dst (NegF src));
14306 
14307   ins_cost(INSN_COST * 3);
14308   format %{ "fneg   $dst, $src" %}
14309 
14310   ins_encode %{
14311     __ fnegs(as_FloatRegister($dst$$reg),
14312              as_FloatRegister($src$$reg));
14313   %}
14314 
14315   ins_pipe(fp_uop_s);
14316 %}
14317 
14318 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14319   match(Set dst (NegD src));
14320 
14321   ins_cost(INSN_COST * 3);
14322   format %{ "fnegd   $dst, $src" %}
14323 
14324   ins_encode %{
14325     __ fnegd(as_FloatRegister($dst$$reg),
14326              as_FloatRegister($src$$reg));
14327   %}
14328 
14329   ins_pipe(fp_uop_d);
14330 %}
14331 
14332 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14333 %{
14334   match(Set dst (AbsI src));
14335 
14336   effect(KILL cr);
14337   ins_cost(INSN_COST * 2);
14338   format %{ "cmpw  $src, zr\n\t"
14339             "cnegw $dst, $src, Assembler::LT\t# int abs"
14340   %}
14341 
14342   ins_encode %{
14343     __ cmpw(as_Register($src$$reg), zr);
14344     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14345   %}
14346   ins_pipe(pipe_class_default);
14347 %}
14348 
14349 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14350 %{
14351   match(Set dst (AbsL src));
14352 
14353   effect(KILL cr);
14354   ins_cost(INSN_COST * 2);
14355   format %{ "cmp  $src, zr\n\t"
14356             "cneg $dst, $src, Assembler::LT\t# long abs"
14357   %}
14358 
14359   ins_encode %{
14360     __ cmp(as_Register($src$$reg), zr);
14361     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14362   %}
14363   ins_pipe(pipe_class_default);
14364 %}
14365 
14366 instruct absF_reg(vRegF dst, vRegF src) %{
14367   match(Set dst (AbsF src));
14368 
14369   ins_cost(INSN_COST * 3);
14370   format %{ "fabss   $dst, $src" %}
14371   ins_encode %{
14372     __ fabss(as_FloatRegister($dst$$reg),
14373              as_FloatRegister($src$$reg));
14374   %}
14375 
14376   ins_pipe(fp_uop_s);
14377 %}
14378 
14379 instruct absD_reg(vRegD dst, vRegD src) %{
14380   match(Set dst (AbsD src));
14381 
14382   ins_cost(INSN_COST * 3);
14383   format %{ "fabsd   $dst, $src" %}
14384   ins_encode %{
14385     __ fabsd(as_FloatRegister($dst$$reg),
14386              as_FloatRegister($src$$reg));
14387   %}
14388 
14389   ins_pipe(fp_uop_d);
14390 %}
14391 
14392 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14393   match(Set dst (AbsF (SubF src1 src2)));
14394 
14395   ins_cost(INSN_COST * 3);
14396   format %{ "fabds   $dst, $src1, $src2" %}
14397   ins_encode %{
14398     __ fabds(as_FloatRegister($dst$$reg),
14399              as_FloatRegister($src1$$reg),
14400              as_FloatRegister($src2$$reg));
14401   %}
14402 
14403   ins_pipe(fp_uop_s);
14404 %}
14405 
14406 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14407   match(Set dst (AbsD (SubD src1 src2)));
14408 
14409   ins_cost(INSN_COST * 3);
14410   format %{ "fabdd   $dst, $src1, $src2" %}
14411   ins_encode %{
14412     __ fabdd(as_FloatRegister($dst$$reg),
14413              as_FloatRegister($src1$$reg),
14414              as_FloatRegister($src2$$reg));
14415   %}
14416 
14417   ins_pipe(fp_uop_d);
14418 %}
14419 
14420 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14421   match(Set dst (SqrtD src));
14422 
14423   ins_cost(INSN_COST * 50);
14424   format %{ "fsqrtd  $dst, $src" %}
14425   ins_encode %{
14426     __ fsqrtd(as_FloatRegister($dst$$reg),
14427              as_FloatRegister($src$$reg));
14428   %}
14429 
14430   ins_pipe(fp_div_s);
14431 %}
14432 
14433 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14434   match(Set dst (SqrtF src));
14435 
14436   ins_cost(INSN_COST * 50);
14437   format %{ "fsqrts  $dst, $src" %}
14438   ins_encode %{
14439     __ fsqrts(as_FloatRegister($dst$$reg),
14440              as_FloatRegister($src$$reg));
14441   %}
14442 
14443   ins_pipe(fp_div_d);
14444 %}
14445 
14446 // Math.rint, floor, ceil
14447 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14448   match(Set dst (RoundDoubleMode src rmode));
14449   format %{ "frint  $dst, $src, $rmode" %}
14450   ins_encode %{
14451     switch ($rmode$$constant) {
14452       case RoundDoubleModeNode::rmode_rint:
14453         __ frintnd(as_FloatRegister($dst$$reg),
14454                    as_FloatRegister($src$$reg));
14455         break;
14456       case RoundDoubleModeNode::rmode_floor:
14457         __ frintmd(as_FloatRegister($dst$$reg),
14458                    as_FloatRegister($src$$reg));
14459         break;
14460       case RoundDoubleModeNode::rmode_ceil:
14461         __ frintpd(as_FloatRegister($dst$$reg),
14462                    as_FloatRegister($src$$reg));
14463         break;
14464     }
14465   %}
14466   ins_pipe(fp_uop_d);
14467 %}
14468 
14469 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14470   match(Set dst (CopySignD src1 (Binary src2 zero)));
14471   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14472   format %{ "CopySignD  $dst $src1 $src2" %}
14473   ins_encode %{
14474     FloatRegister dst = as_FloatRegister($dst$$reg),
14475                   src1 = as_FloatRegister($src1$$reg),
14476                   src2 = as_FloatRegister($src2$$reg),
14477                   zero = as_FloatRegister($zero$$reg);
14478     __ fnegd(dst, zero);
14479     __ bsl(dst, __ T8B, src2, src1);
14480   %}
14481   ins_pipe(fp_uop_d);
14482 %}
14483 
14484 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14485   match(Set dst (CopySignF src1 src2));
14486   effect(TEMP_DEF dst, USE src1, USE src2);
14487   format %{ "CopySignF  $dst $src1 $src2" %}
14488   ins_encode %{
14489     FloatRegister dst = as_FloatRegister($dst$$reg),
14490                   src1 = as_FloatRegister($src1$$reg),
14491                   src2 = as_FloatRegister($src2$$reg);
14492     __ movi(dst, __ T2S, 0x80, 24);
14493     __ bsl(dst, __ T8B, src2, src1);
14494   %}
14495   ins_pipe(fp_uop_d);
14496 %}
14497 
14498 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14499   match(Set dst (SignumD src (Binary zero one)));
14500   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14501   format %{ "signumD  $dst, $src" %}
14502   ins_encode %{
14503     FloatRegister src = as_FloatRegister($src$$reg),
14504                   dst = as_FloatRegister($dst$$reg),
14505                   zero = as_FloatRegister($zero$$reg),
14506                   one = as_FloatRegister($one$$reg);
14507     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14508     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14509     // Bit selection instruction gets bit from "one" for each enabled bit in
14510     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14511     // NaN the whole "src" will be copied because "dst" is zero. For all other
14512     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14513     // from "src", and all other bits are copied from 1.0.
14514     __ bsl(dst, __ T8B, one, src);
14515   %}
14516   ins_pipe(fp_uop_d);
14517 %}
14518 
14519 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14520   match(Set dst (SignumF src (Binary zero one)));
14521   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14522   format %{ "signumF  $dst, $src" %}
14523   ins_encode %{
14524     FloatRegister src = as_FloatRegister($src$$reg),
14525                   dst = as_FloatRegister($dst$$reg),
14526                   zero = as_FloatRegister($zero$$reg),
14527                   one = as_FloatRegister($one$$reg);
14528     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14529     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14530     // Bit selection instruction gets bit from "one" for each enabled bit in
14531     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14532     // NaN the whole "src" will be copied because "dst" is zero. For all other
14533     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14534     // from "src", and all other bits are copied from 1.0.
14535     __ bsl(dst, __ T8B, one, src);
14536   %}
14537   ins_pipe(fp_uop_d);
14538 %}
14539 
14540 instruct onspinwait() %{
14541   match(OnSpinWait);
14542   ins_cost(INSN_COST);
14543 
14544   format %{ "onspinwait" %}
14545 
14546   ins_encode %{
14547     __ spin_wait();
14548   %}
14549   ins_pipe(pipe_class_empty);
14550 %}
14551 
14552 // ============================================================================
14553 // Logical Instructions
14554 
14555 // Integer Logical Instructions
14556 
14557 // And Instructions
14558 
14559 
14560 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14561   match(Set dst (AndI src1 src2));
14562 
14563   format %{ "andw  $dst, $src1, $src2\t# int" %}
14564 
14565   ins_cost(INSN_COST);
14566   ins_encode %{
14567     __ andw(as_Register($dst$$reg),
14568             as_Register($src1$$reg),
14569             as_Register($src2$$reg));
14570   %}
14571 
14572   ins_pipe(ialu_reg_reg);
14573 %}
14574 
14575 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14576   match(Set dst (AndI src1 src2));
14577 
14578   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14579 
14580   ins_cost(INSN_COST);
14581   ins_encode %{
14582     __ andw(as_Register($dst$$reg),
14583             as_Register($src1$$reg),
14584             (uint64_t)($src2$$constant));
14585   %}
14586 
14587   ins_pipe(ialu_reg_imm);
14588 %}
14589 
14590 // Or Instructions
14591 
14592 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14593   match(Set dst (OrI src1 src2));
14594 
14595   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14596 
14597   ins_cost(INSN_COST);
14598   ins_encode %{
14599     __ orrw(as_Register($dst$$reg),
14600             as_Register($src1$$reg),
14601             as_Register($src2$$reg));
14602   %}
14603 
14604   ins_pipe(ialu_reg_reg);
14605 %}
14606 
14607 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14608   match(Set dst (OrI src1 src2));
14609 
14610   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14611 
14612   ins_cost(INSN_COST);
14613   ins_encode %{
14614     __ orrw(as_Register($dst$$reg),
14615             as_Register($src1$$reg),
14616             (uint64_t)($src2$$constant));
14617   %}
14618 
14619   ins_pipe(ialu_reg_imm);
14620 %}
14621 
14622 // Xor Instructions
14623 
14624 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14625   match(Set dst (XorI src1 src2));
14626 
14627   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14628 
14629   ins_cost(INSN_COST);
14630   ins_encode %{
14631     __ eorw(as_Register($dst$$reg),
14632             as_Register($src1$$reg),
14633             as_Register($src2$$reg));
14634   %}
14635 
14636   ins_pipe(ialu_reg_reg);
14637 %}
14638 
14639 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14640   match(Set dst (XorI src1 src2));
14641 
14642   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14643 
14644   ins_cost(INSN_COST);
14645   ins_encode %{
14646     __ eorw(as_Register($dst$$reg),
14647             as_Register($src1$$reg),
14648             (uint64_t)($src2$$constant));
14649   %}
14650 
14651   ins_pipe(ialu_reg_imm);
14652 %}
14653 
14654 // Long Logical Instructions
14655 // TODO
14656 
14657 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14658   match(Set dst (AndL src1 src2));
14659 
14660   format %{ "and  $dst, $src1, $src2\t# int" %}
14661 
14662   ins_cost(INSN_COST);
14663   ins_encode %{
14664     __ andr(as_Register($dst$$reg),
14665             as_Register($src1$$reg),
14666             as_Register($src2$$reg));
14667   %}
14668 
14669   ins_pipe(ialu_reg_reg);
14670 %}
14671 
14672 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14673   match(Set dst (AndL src1 src2));
14674 
14675   format %{ "and  $dst, $src1, $src2\t# int" %}
14676 
14677   ins_cost(INSN_COST);
14678   ins_encode %{
14679     __ andr(as_Register($dst$$reg),
14680             as_Register($src1$$reg),
14681             (uint64_t)($src2$$constant));
14682   %}
14683 
14684   ins_pipe(ialu_reg_imm);
14685 %}
14686 
14687 // Or Instructions
14688 
14689 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14690   match(Set dst (OrL src1 src2));
14691 
14692   format %{ "orr  $dst, $src1, $src2\t# int" %}
14693 
14694   ins_cost(INSN_COST);
14695   ins_encode %{
14696     __ orr(as_Register($dst$$reg),
14697            as_Register($src1$$reg),
14698            as_Register($src2$$reg));
14699   %}
14700 
14701   ins_pipe(ialu_reg_reg);
14702 %}
14703 
14704 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14705   match(Set dst (OrL src1 src2));
14706 
14707   format %{ "orr  $dst, $src1, $src2\t# int" %}
14708 
14709   ins_cost(INSN_COST);
14710   ins_encode %{
14711     __ orr(as_Register($dst$$reg),
14712            as_Register($src1$$reg),
14713            (uint64_t)($src2$$constant));
14714   %}
14715 
14716   ins_pipe(ialu_reg_imm);
14717 %}
14718 
14719 // Xor Instructions
14720 
14721 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14722   match(Set dst (XorL src1 src2));
14723 
14724   format %{ "eor  $dst, $src1, $src2\t# int" %}
14725 
14726   ins_cost(INSN_COST);
14727   ins_encode %{
14728     __ eor(as_Register($dst$$reg),
14729            as_Register($src1$$reg),
14730            as_Register($src2$$reg));
14731   %}
14732 
14733   ins_pipe(ialu_reg_reg);
14734 %}
14735 
14736 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14737   match(Set dst (XorL src1 src2));
14738 
14739   ins_cost(INSN_COST);
14740   format %{ "eor  $dst, $src1, $src2\t# int" %}
14741 
14742   ins_encode %{
14743     __ eor(as_Register($dst$$reg),
14744            as_Register($src1$$reg),
14745            (uint64_t)($src2$$constant));
14746   %}
14747 
14748   ins_pipe(ialu_reg_imm);
14749 %}
14750 
14751 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14752 %{
14753   match(Set dst (ConvI2L src));
14754 
14755   ins_cost(INSN_COST);
14756   format %{ "sxtw  $dst, $src\t# i2l" %}
14757   ins_encode %{
14758     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14759   %}
14760   ins_pipe(ialu_reg_shift);
14761 %}
14762 
14763 // this pattern occurs in bigmath arithmetic
14764 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14765 %{
14766   match(Set dst (AndL (ConvI2L src) mask));
14767 
14768   ins_cost(INSN_COST);
14769   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14770   ins_encode %{
14771     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14772   %}
14773 
14774   ins_pipe(ialu_reg_shift);
14775 %}
14776 
14777 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14778   match(Set dst (ConvL2I src));
14779 
14780   ins_cost(INSN_COST);
14781   format %{ "movw  $dst, $src \t// l2i" %}
14782 
14783   ins_encode %{
14784     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14785   %}
14786 
14787   ins_pipe(ialu_reg);
14788 %}
14789 
14790 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14791 %{
14792   match(Set dst (Conv2B src));
14793   effect(KILL cr);
14794 
14795   format %{
14796     "cmpw $src, zr\n\t"
14797     "cset $dst, ne"
14798   %}
14799 
14800   ins_encode %{
14801     __ cmpw(as_Register($src$$reg), zr);
14802     __ cset(as_Register($dst$$reg), Assembler::NE);
14803   %}
14804 
14805   ins_pipe(ialu_reg);
14806 %}
14807 
14808 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14809 %{
14810   match(Set dst (Conv2B src));
14811   effect(KILL cr);
14812 
14813   format %{
14814     "cmp  $src, zr\n\t"
14815     "cset $dst, ne"
14816   %}
14817 
14818   ins_encode %{
14819     __ cmp(as_Register($src$$reg), zr);
14820     __ cset(as_Register($dst$$reg), Assembler::NE);
14821   %}
14822 
14823   ins_pipe(ialu_reg);
14824 %}
14825 
14826 instruct convD2F_reg(vRegF dst, vRegD src) %{
14827   match(Set dst (ConvD2F src));
14828 
14829   ins_cost(INSN_COST * 5);
14830   format %{ "fcvtd  $dst, $src \t// d2f" %}
14831 
14832   ins_encode %{
14833     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14834   %}
14835 
14836   ins_pipe(fp_d2f);
14837 %}
14838 
14839 instruct convF2D_reg(vRegD dst, vRegF src) %{
14840   match(Set dst (ConvF2D src));
14841 
14842   ins_cost(INSN_COST * 5);
14843   format %{ "fcvts  $dst, $src \t// f2d" %}
14844 
14845   ins_encode %{
14846     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14847   %}
14848 
14849   ins_pipe(fp_f2d);
14850 %}
14851 
14852 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14853   match(Set dst (ConvF2I src));
14854 
14855   ins_cost(INSN_COST * 5);
14856   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14857 
14858   ins_encode %{
14859     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14860   %}
14861 
14862   ins_pipe(fp_f2i);
14863 %}
14864 
14865 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14866   match(Set dst (ConvF2L src));
14867 
14868   ins_cost(INSN_COST * 5);
14869   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14870 
14871   ins_encode %{
14872     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14873   %}
14874 
14875   ins_pipe(fp_f2l);
14876 %}
14877 
14878 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14879   match(Set dst (ConvI2F src));
14880 
14881   ins_cost(INSN_COST * 5);
14882   format %{ "scvtfws  $dst, $src \t// i2f" %}
14883 
14884   ins_encode %{
14885     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14886   %}
14887 
14888   ins_pipe(fp_i2f);
14889 %}
14890 
14891 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14892   match(Set dst (ConvL2F src));
14893 
14894   ins_cost(INSN_COST * 5);
14895   format %{ "scvtfs  $dst, $src \t// l2f" %}
14896 
14897   ins_encode %{
14898     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14899   %}
14900 
14901   ins_pipe(fp_l2f);
14902 %}
14903 
14904 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14905   match(Set dst (ConvD2I src));
14906 
14907   ins_cost(INSN_COST * 5);
14908   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14909 
14910   ins_encode %{
14911     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14912   %}
14913 
14914   ins_pipe(fp_d2i);
14915 %}
14916 
14917 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14918   match(Set dst (ConvD2L src));
14919 
14920   ins_cost(INSN_COST * 5);
14921   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14922 
14923   ins_encode %{
14924     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14925   %}
14926 
14927   ins_pipe(fp_d2l);
14928 %}
14929 
14930 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14931   match(Set dst (ConvI2D src));
14932 
14933   ins_cost(INSN_COST * 5);
14934   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14935 
14936   ins_encode %{
14937     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14938   %}
14939 
14940   ins_pipe(fp_i2d);
14941 %}
14942 
14943 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14944   match(Set dst (ConvL2D src));
14945 
14946   ins_cost(INSN_COST * 5);
14947   format %{ "scvtfd  $dst, $src \t// l2d" %}
14948 
14949   ins_encode %{
14950     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14951   %}
14952 
14953   ins_pipe(fp_l2d);
14954 %}
14955 
14956 // stack <-> reg and reg <-> reg shuffles with no conversion
14957 
14958 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14959 
14960   match(Set dst (MoveF2I src));
14961 
14962   effect(DEF dst, USE src);
14963 
14964   ins_cost(4 * INSN_COST);
14965 
14966   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14967 
14968   ins_encode %{
14969     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14970   %}
14971 
14972   ins_pipe(iload_reg_reg);
14973 
14974 %}
14975 
14976 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14977 
14978   match(Set dst (MoveI2F src));
14979 
14980   effect(DEF dst, USE src);
14981 
14982   ins_cost(4 * INSN_COST);
14983 
14984   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14985 
14986   ins_encode %{
14987     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14988   %}
14989 
14990   ins_pipe(pipe_class_memory);
14991 
14992 %}
14993 
14994 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14995 
14996   match(Set dst (MoveD2L src));
14997 
14998   effect(DEF dst, USE src);
14999 
15000   ins_cost(4 * INSN_COST);
15001 
15002   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15003 
15004   ins_encode %{
15005     __ ldr($dst$$Register, Address(sp, $src$$disp));
15006   %}
15007 
15008   ins_pipe(iload_reg_reg);
15009 
15010 %}
15011 
15012 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15013 
15014   match(Set dst (MoveL2D src));
15015 
15016   effect(DEF dst, USE src);
15017 
15018   ins_cost(4 * INSN_COST);
15019 
15020   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15021 
15022   ins_encode %{
15023     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15024   %}
15025 
15026   ins_pipe(pipe_class_memory);
15027 
15028 %}
15029 
15030 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15031 
15032   match(Set dst (MoveF2I src));
15033 
15034   effect(DEF dst, USE src);
15035 
15036   ins_cost(INSN_COST);
15037 
15038   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15039 
15040   ins_encode %{
15041     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15042   %}
15043 
15044   ins_pipe(pipe_class_memory);
15045 
15046 %}
15047 
15048 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15049 
15050   match(Set dst (MoveI2F src));
15051 
15052   effect(DEF dst, USE src);
15053 
15054   ins_cost(INSN_COST);
15055 
15056   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15057 
15058   ins_encode %{
15059     __ strw($src$$Register, Address(sp, $dst$$disp));
15060   %}
15061 
15062   ins_pipe(istore_reg_reg);
15063 
15064 %}
15065 
15066 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15067 
15068   match(Set dst (MoveD2L src));
15069 
15070   effect(DEF dst, USE src);
15071 
15072   ins_cost(INSN_COST);
15073 
15074   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15075 
15076   ins_encode %{
15077     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15078   %}
15079 
15080   ins_pipe(pipe_class_memory);
15081 
15082 %}
15083 
15084 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15085 
15086   match(Set dst (MoveL2D src));
15087 
15088   effect(DEF dst, USE src);
15089 
15090   ins_cost(INSN_COST);
15091 
15092   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15093 
15094   ins_encode %{
15095     __ str($src$$Register, Address(sp, $dst$$disp));
15096   %}
15097 
15098   ins_pipe(istore_reg_reg);
15099 
15100 %}
15101 
15102 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15103 
15104   match(Set dst (MoveF2I src));
15105 
15106   effect(DEF dst, USE src);
15107 
15108   ins_cost(INSN_COST);
15109 
15110   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15111 
15112   ins_encode %{
15113     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15114   %}
15115 
15116   ins_pipe(fp_f2i);
15117 
15118 %}
15119 
15120 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15121 
15122   match(Set dst (MoveI2F src));
15123 
15124   effect(DEF dst, USE src);
15125 
15126   ins_cost(INSN_COST);
15127 
15128   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15129 
15130   ins_encode %{
15131     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15132   %}
15133 
15134   ins_pipe(fp_i2f);
15135 
15136 %}
15137 
15138 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15139 
15140   match(Set dst (MoveD2L src));
15141 
15142   effect(DEF dst, USE src);
15143 
15144   ins_cost(INSN_COST);
15145 
15146   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15147 
15148   ins_encode %{
15149     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15150   %}
15151 
15152   ins_pipe(fp_d2l);
15153 
15154 %}
15155 
15156 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15157 
15158   match(Set dst (MoveL2D src));
15159 
15160   effect(DEF dst, USE src);
15161 
15162   ins_cost(INSN_COST);
15163 
15164   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15165 
15166   ins_encode %{
15167     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15168   %}
15169 
15170   ins_pipe(fp_l2d);
15171 
15172 %}
15173 
15174 // ============================================================================
15175 // clearing of an array
15176 
15177 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15178 %{
15179   match(Set dummy (ClearArray cnt base));
15180   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15181 
15182   ins_cost(4 * INSN_COST);
15183   format %{ "ClearArray $cnt, $base" %}
15184 
15185   ins_encode %{
15186     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15187     if (tpc == NULL) {
15188       ciEnv::current()->record_failure("CodeCache is full");
15189       return;
15190     }
15191   %}
15192 
15193   ins_pipe(pipe_class_memory);
15194 %}
15195 
15196 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15197 %{
15198   predicate((uint64_t)n->in(2)->get_long()
15199             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15200   match(Set dummy (ClearArray cnt base));
15201   effect(TEMP temp, USE_KILL base, KILL cr);
15202 
15203   ins_cost(4 * INSN_COST);
15204   format %{ "ClearArray $cnt, $base" %}
15205 
15206   ins_encode %{
15207     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15208   %}
15209 
15210   ins_pipe(pipe_class_memory);
15211 %}
15212 
15213 // ============================================================================
15214 // Overflow Math Instructions
15215 
15216 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15217 %{
15218   match(Set cr (OverflowAddI op1 op2));
15219 
15220   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15221   ins_cost(INSN_COST);
15222   ins_encode %{
15223     __ cmnw($op1$$Register, $op2$$Register);
15224   %}
15225 
15226   ins_pipe(icmp_reg_reg);
15227 %}
15228 
15229 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15230 %{
15231   match(Set cr (OverflowAddI op1 op2));
15232 
15233   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15234   ins_cost(INSN_COST);
15235   ins_encode %{
15236     __ cmnw($op1$$Register, $op2$$constant);
15237   %}
15238 
15239   ins_pipe(icmp_reg_imm);
15240 %}
15241 
15242 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15243 %{
15244   match(Set cr (OverflowAddL op1 op2));
15245 
15246   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15247   ins_cost(INSN_COST);
15248   ins_encode %{
15249     __ cmn($op1$$Register, $op2$$Register);
15250   %}
15251 
15252   ins_pipe(icmp_reg_reg);
15253 %}
15254 
15255 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15256 %{
15257   match(Set cr (OverflowAddL op1 op2));
15258 
15259   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15260   ins_cost(INSN_COST);
15261   ins_encode %{
15262     __ cmn($op1$$Register, $op2$$constant);
15263   %}
15264 
15265   ins_pipe(icmp_reg_imm);
15266 %}
15267 
15268 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15269 %{
15270   match(Set cr (OverflowSubI op1 op2));
15271 
15272   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15273   ins_cost(INSN_COST);
15274   ins_encode %{
15275     __ cmpw($op1$$Register, $op2$$Register);
15276   %}
15277 
15278   ins_pipe(icmp_reg_reg);
15279 %}
15280 
15281 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15282 %{
15283   match(Set cr (OverflowSubI op1 op2));
15284 
15285   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15286   ins_cost(INSN_COST);
15287   ins_encode %{
15288     __ cmpw($op1$$Register, $op2$$constant);
15289   %}
15290 
15291   ins_pipe(icmp_reg_imm);
15292 %}
15293 
15294 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15295 %{
15296   match(Set cr (OverflowSubL op1 op2));
15297 
15298   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15299   ins_cost(INSN_COST);
15300   ins_encode %{
15301     __ cmp($op1$$Register, $op2$$Register);
15302   %}
15303 
15304   ins_pipe(icmp_reg_reg);
15305 %}
15306 
15307 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15308 %{
15309   match(Set cr (OverflowSubL op1 op2));
15310 
15311   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15312   ins_cost(INSN_COST);
15313   ins_encode %{
15314     __ subs(zr, $op1$$Register, $op2$$constant);
15315   %}
15316 
15317   ins_pipe(icmp_reg_imm);
15318 %}
15319 
15320 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15321 %{
15322   match(Set cr (OverflowSubI zero op1));
15323 
15324   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15325   ins_cost(INSN_COST);
15326   ins_encode %{
15327     __ cmpw(zr, $op1$$Register);
15328   %}
15329 
15330   ins_pipe(icmp_reg_imm);
15331 %}
15332 
15333 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15334 %{
15335   match(Set cr (OverflowSubL zero op1));
15336 
15337   format %{ "cmp   zr, $op1\t# overflow check long" %}
15338   ins_cost(INSN_COST);
15339   ins_encode %{
15340     __ cmp(zr, $op1$$Register);
15341   %}
15342 
15343   ins_pipe(icmp_reg_imm);
15344 %}
15345 
15346 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15347 %{
15348   match(Set cr (OverflowMulI op1 op2));
15349 
15350   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15351             "cmp   rscratch1, rscratch1, sxtw\n\t"
15352             "movw  rscratch1, #0x80000000\n\t"
15353             "cselw rscratch1, rscratch1, zr, NE\n\t"
15354             "cmpw  rscratch1, #1" %}
15355   ins_cost(5 * INSN_COST);
15356   ins_encode %{
15357     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15358     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15359     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15360     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15361     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15362   %}
15363 
15364   ins_pipe(pipe_slow);
15365 %}
15366 
15367 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15368 %{
15369   match(If cmp (OverflowMulI op1 op2));
15370   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15371             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15372   effect(USE labl, KILL cr);
15373 
15374   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15375             "cmp   rscratch1, rscratch1, sxtw\n\t"
15376             "b$cmp   $labl" %}
15377   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15378   ins_encode %{
15379     Label* L = $labl$$label;
15380     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15381     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15382     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15383     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15384   %}
15385 
15386   ins_pipe(pipe_serial);
15387 %}
15388 
15389 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15390 %{
15391   match(Set cr (OverflowMulL op1 op2));
15392 
15393   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15394             "smulh rscratch2, $op1, $op2\n\t"
15395             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15396             "movw  rscratch1, #0x80000000\n\t"
15397             "cselw rscratch1, rscratch1, zr, NE\n\t"
15398             "cmpw  rscratch1, #1" %}
15399   ins_cost(6 * INSN_COST);
15400   ins_encode %{
15401     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15402     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15403     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15404     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15405     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15406     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15407   %}
15408 
15409   ins_pipe(pipe_slow);
15410 %}
15411 
15412 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15413 %{
15414   match(If cmp (OverflowMulL op1 op2));
15415   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15416             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15417   effect(USE labl, KILL cr);
15418 
15419   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15420             "smulh rscratch2, $op1, $op2\n\t"
15421             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15422             "b$cmp $labl" %}
15423   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15424   ins_encode %{
15425     Label* L = $labl$$label;
15426     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15427     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15428     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15429     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15430     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15431   %}
15432 
15433   ins_pipe(pipe_serial);
15434 %}
15435 
15436 // ============================================================================
15437 // Compare Instructions
15438 
15439 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15440 %{
15441   match(Set cr (CmpI op1 op2));
15442 
15443   effect(DEF cr, USE op1, USE op2);
15444 
15445   ins_cost(INSN_COST);
15446   format %{ "cmpw  $op1, $op2" %}
15447 
15448   ins_encode(aarch64_enc_cmpw(op1, op2));
15449 
15450   ins_pipe(icmp_reg_reg);
15451 %}
15452 
15453 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15454 %{
15455   match(Set cr (CmpI op1 zero));
15456 
15457   effect(DEF cr, USE op1);
15458 
15459   ins_cost(INSN_COST);
15460   format %{ "cmpw $op1, 0" %}
15461 
15462   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15463 
15464   ins_pipe(icmp_reg_imm);
15465 %}
15466 
15467 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15468 %{
15469   match(Set cr (CmpI op1 op2));
15470 
15471   effect(DEF cr, USE op1);
15472 
15473   ins_cost(INSN_COST);
15474   format %{ "cmpw  $op1, $op2" %}
15475 
15476   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15477 
15478   ins_pipe(icmp_reg_imm);
15479 %}
15480 
15481 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15482 %{
15483   match(Set cr (CmpI op1 op2));
15484 
15485   effect(DEF cr, USE op1);
15486 
15487   ins_cost(INSN_COST * 2);
15488   format %{ "cmpw  $op1, $op2" %}
15489 
15490   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15491 
15492   ins_pipe(icmp_reg_imm);
15493 %}
15494 
15495 // Unsigned compare Instructions; really, same as signed compare
15496 // except it should only be used to feed an If or a CMovI which takes a
15497 // cmpOpU.
15498 
15499 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15500 %{
15501   match(Set cr (CmpU op1 op2));
15502 
15503   effect(DEF cr, USE op1, USE op2);
15504 
15505   ins_cost(INSN_COST);
15506   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15507 
15508   ins_encode(aarch64_enc_cmpw(op1, op2));
15509 
15510   ins_pipe(icmp_reg_reg);
15511 %}
15512 
15513 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15514 %{
15515   match(Set cr (CmpU op1 zero));
15516 
15517   effect(DEF cr, USE op1);
15518 
15519   ins_cost(INSN_COST);
15520   format %{ "cmpw $op1, #0\t# unsigned" %}
15521 
15522   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15523 
15524   ins_pipe(icmp_reg_imm);
15525 %}
15526 
15527 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15528 %{
15529   match(Set cr (CmpU op1 op2));
15530 
15531   effect(DEF cr, USE op1);
15532 
15533   ins_cost(INSN_COST);
15534   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15535 
15536   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15537 
15538   ins_pipe(icmp_reg_imm);
15539 %}
15540 
15541 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15542 %{
15543   match(Set cr (CmpU op1 op2));
15544 
15545   effect(DEF cr, USE op1);
15546 
15547   ins_cost(INSN_COST * 2);
15548   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15549 
15550   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15551 
15552   ins_pipe(icmp_reg_imm);
15553 %}
15554 
15555 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15556 %{
15557   match(Set cr (CmpL op1 op2));
15558 
15559   effect(DEF cr, USE op1, USE op2);
15560 
15561   ins_cost(INSN_COST);
15562   format %{ "cmp  $op1, $op2" %}
15563 
15564   ins_encode(aarch64_enc_cmp(op1, op2));
15565 
15566   ins_pipe(icmp_reg_reg);
15567 %}
15568 
15569 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15570 %{
15571   match(Set cr (CmpL op1 zero));
15572 
15573   effect(DEF cr, USE op1);
15574 
15575   ins_cost(INSN_COST);
15576   format %{ "tst  $op1" %}
15577 
15578   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15579 
15580   ins_pipe(icmp_reg_imm);
15581 %}
15582 
15583 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15584 %{
15585   match(Set cr (CmpL op1 op2));
15586 
15587   effect(DEF cr, USE op1);
15588 
15589   ins_cost(INSN_COST);
15590   format %{ "cmp  $op1, $op2" %}
15591 
15592   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15593 
15594   ins_pipe(icmp_reg_imm);
15595 %}
15596 
15597 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15598 %{
15599   match(Set cr (CmpL op1 op2));
15600 
15601   effect(DEF cr, USE op1);
15602 
15603   ins_cost(INSN_COST * 2);
15604   format %{ "cmp  $op1, $op2" %}
15605 
15606   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15607 
15608   ins_pipe(icmp_reg_imm);
15609 %}
15610 
15611 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15612 %{
15613   match(Set cr (CmpUL op1 op2));
15614 
15615   effect(DEF cr, USE op1, USE op2);
15616 
15617   ins_cost(INSN_COST);
15618   format %{ "cmp  $op1, $op2" %}
15619 
15620   ins_encode(aarch64_enc_cmp(op1, op2));
15621 
15622   ins_pipe(icmp_reg_reg);
15623 %}
15624 
15625 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15626 %{
15627   match(Set cr (CmpUL op1 zero));
15628 
15629   effect(DEF cr, USE op1);
15630 
15631   ins_cost(INSN_COST);
15632   format %{ "tst  $op1" %}
15633 
15634   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15635 
15636   ins_pipe(icmp_reg_imm);
15637 %}
15638 
15639 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15640 %{
15641   match(Set cr (CmpUL op1 op2));
15642 
15643   effect(DEF cr, USE op1);
15644 
15645   ins_cost(INSN_COST);
15646   format %{ "cmp  $op1, $op2" %}
15647 
15648   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15649 
15650   ins_pipe(icmp_reg_imm);
15651 %}
15652 
15653 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15654 %{
15655   match(Set cr (CmpUL op1 op2));
15656 
15657   effect(DEF cr, USE op1);
15658 
15659   ins_cost(INSN_COST * 2);
15660   format %{ "cmp  $op1, $op2" %}
15661 
15662   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15663 
15664   ins_pipe(icmp_reg_imm);
15665 %}
15666 
15667 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15668 %{
15669   match(Set cr (CmpP op1 op2));
15670 
15671   effect(DEF cr, USE op1, USE op2);
15672 
15673   ins_cost(INSN_COST);
15674   format %{ "cmp  $op1, $op2\t // ptr" %}
15675 
15676   ins_encode(aarch64_enc_cmpp(op1, op2));
15677 
15678   ins_pipe(icmp_reg_reg);
15679 %}
15680 
15681 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15682 %{
15683   match(Set cr (CmpN op1 op2));
15684 
15685   effect(DEF cr, USE op1, USE op2);
15686 
15687   ins_cost(INSN_COST);
15688   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15689 
15690   ins_encode(aarch64_enc_cmpn(op1, op2));
15691 
15692   ins_pipe(icmp_reg_reg);
15693 %}
15694 
15695 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15696 %{
15697   match(Set cr (CmpP op1 zero));
15698 
15699   effect(DEF cr, USE op1, USE zero);
15700 
15701   ins_cost(INSN_COST);
15702   format %{ "cmp  $op1, 0\t // ptr" %}
15703 
15704   ins_encode(aarch64_enc_testp(op1));
15705 
15706   ins_pipe(icmp_reg_imm);
15707 %}
15708 
15709 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15710 %{
15711   match(Set cr (CmpN op1 zero));
15712 
15713   effect(DEF cr, USE op1, USE zero);
15714 
15715   ins_cost(INSN_COST);
15716   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15717 
15718   ins_encode(aarch64_enc_testn(op1));
15719 
15720   ins_pipe(icmp_reg_imm);
15721 %}
15722 
15723 // FP comparisons
15724 //
15725 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15726 // using normal cmpOp. See declaration of rFlagsReg for details.
15727 
15728 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15729 %{
15730   match(Set cr (CmpF src1 src2));
15731 
15732   ins_cost(3 * INSN_COST);
15733   format %{ "fcmps $src1, $src2" %}
15734 
15735   ins_encode %{
15736     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15737   %}
15738 
15739   ins_pipe(pipe_class_compare);
15740 %}
15741 
15742 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15743 %{
15744   match(Set cr (CmpF src1 src2));
15745 
15746   ins_cost(3 * INSN_COST);
15747   format %{ "fcmps $src1, 0.0" %}
15748 
15749   ins_encode %{
15750     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15751   %}
15752 
15753   ins_pipe(pipe_class_compare);
15754 %}
15755 // FROM HERE
15756 
15757 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15758 %{
15759   match(Set cr (CmpD src1 src2));
15760 
15761   ins_cost(3 * INSN_COST);
15762   format %{ "fcmpd $src1, $src2" %}
15763 
15764   ins_encode %{
15765     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15766   %}
15767 
15768   ins_pipe(pipe_class_compare);
15769 %}
15770 
15771 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15772 %{
15773   match(Set cr (CmpD src1 src2));
15774 
15775   ins_cost(3 * INSN_COST);
15776   format %{ "fcmpd $src1, 0.0" %}
15777 
15778   ins_encode %{
15779     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15780   %}
15781 
15782   ins_pipe(pipe_class_compare);
15783 %}
15784 
15785 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15786 %{
15787   match(Set dst (CmpF3 src1 src2));
15788   effect(KILL cr);
15789 
15790   ins_cost(5 * INSN_COST);
15791   format %{ "fcmps $src1, $src2\n\t"
15792             "csinvw($dst, zr, zr, eq\n\t"
15793             "csnegw($dst, $dst, $dst, lt)"
15794   %}
15795 
15796   ins_encode %{
15797     Label done;
15798     FloatRegister s1 = as_FloatRegister($src1$$reg);
15799     FloatRegister s2 = as_FloatRegister($src2$$reg);
15800     Register d = as_Register($dst$$reg);
15801     __ fcmps(s1, s2);
15802     // installs 0 if EQ else -1
15803     __ csinvw(d, zr, zr, Assembler::EQ);
15804     // keeps -1 if less or unordered else installs 1
15805     __ csnegw(d, d, d, Assembler::LT);
15806     __ bind(done);
15807   %}
15808 
15809   ins_pipe(pipe_class_default);
15810 
15811 %}
15812 
15813 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15814 %{
15815   match(Set dst (CmpD3 src1 src2));
15816   effect(KILL cr);
15817 
15818   ins_cost(5 * INSN_COST);
15819   format %{ "fcmpd $src1, $src2\n\t"
15820             "csinvw($dst, zr, zr, eq\n\t"
15821             "csnegw($dst, $dst, $dst, lt)"
15822   %}
15823 
15824   ins_encode %{
15825     Label done;
15826     FloatRegister s1 = as_FloatRegister($src1$$reg);
15827     FloatRegister s2 = as_FloatRegister($src2$$reg);
15828     Register d = as_Register($dst$$reg);
15829     __ fcmpd(s1, s2);
15830     // installs 0 if EQ else -1
15831     __ csinvw(d, zr, zr, Assembler::EQ);
15832     // keeps -1 if less or unordered else installs 1
15833     __ csnegw(d, d, d, Assembler::LT);
15834     __ bind(done);
15835   %}
15836   ins_pipe(pipe_class_default);
15837 
15838 %}
15839 
15840 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15841 %{
15842   match(Set dst (CmpF3 src1 zero));
15843   effect(KILL cr);
15844 
15845   ins_cost(5 * INSN_COST);
15846   format %{ "fcmps $src1, 0.0\n\t"
15847             "csinvw($dst, zr, zr, eq\n\t"
15848             "csnegw($dst, $dst, $dst, lt)"
15849   %}
15850 
15851   ins_encode %{
15852     Label done;
15853     FloatRegister s1 = as_FloatRegister($src1$$reg);
15854     Register d = as_Register($dst$$reg);
15855     __ fcmps(s1, 0.0);
15856     // installs 0 if EQ else -1
15857     __ csinvw(d, zr, zr, Assembler::EQ);
15858     // keeps -1 if less or unordered else installs 1
15859     __ csnegw(d, d, d, Assembler::LT);
15860     __ bind(done);
15861   %}
15862 
15863   ins_pipe(pipe_class_default);
15864 
15865 %}
15866 
15867 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15868 %{
15869   match(Set dst (CmpD3 src1 zero));
15870   effect(KILL cr);
15871 
15872   ins_cost(5 * INSN_COST);
15873   format %{ "fcmpd $src1, 0.0\n\t"
15874             "csinvw($dst, zr, zr, eq\n\t"
15875             "csnegw($dst, $dst, $dst, lt)"
15876   %}
15877 
15878   ins_encode %{
15879     Label done;
15880     FloatRegister s1 = as_FloatRegister($src1$$reg);
15881     Register d = as_Register($dst$$reg);
15882     __ fcmpd(s1, 0.0);
15883     // installs 0 if EQ else -1
15884     __ csinvw(d, zr, zr, Assembler::EQ);
15885     // keeps -1 if less or unordered else installs 1
15886     __ csnegw(d, d, d, Assembler::LT);
15887     __ bind(done);
15888   %}
15889   ins_pipe(pipe_class_default);
15890 
15891 %}
15892 
15893 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15894 %{
15895   match(Set dst (CmpLTMask p q));
15896   effect(KILL cr);
15897 
15898   ins_cost(3 * INSN_COST);
15899 
15900   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15901             "csetw $dst, lt\n\t"
15902             "subw $dst, zr, $dst"
15903   %}
15904 
15905   ins_encode %{
15906     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15907     __ csetw(as_Register($dst$$reg), Assembler::LT);
15908     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15909   %}
15910 
15911   ins_pipe(ialu_reg_reg);
15912 %}
15913 
15914 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15915 %{
15916   match(Set dst (CmpLTMask src zero));
15917   effect(KILL cr);
15918 
15919   ins_cost(INSN_COST);
15920 
15921   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15922 
15923   ins_encode %{
15924     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15925   %}
15926 
15927   ins_pipe(ialu_reg_shift);
15928 %}
15929 
15930 // ============================================================================
15931 // Max and Min
15932 
15933 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15934 %{
15935   effect( DEF dst, USE src1, USE src2, USE cr );
15936 
15937   ins_cost(INSN_COST * 2);
15938   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15939 
15940   ins_encode %{
15941     __ cselw(as_Register($dst$$reg),
15942              as_Register($src1$$reg),
15943              as_Register($src2$$reg),
15944              Assembler::LT);
15945   %}
15946 
15947   ins_pipe(icond_reg_reg);
15948 %}
15949 
15950 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15951 %{
15952   match(Set dst (MinI src1 src2));
15953   ins_cost(INSN_COST * 3);
15954 
15955   expand %{
15956     rFlagsReg cr;
15957     compI_reg_reg(cr, src1, src2);
15958     cmovI_reg_reg_lt(dst, src1, src2, cr);
15959   %}
15960 
15961 %}
15962 // FROM HERE
15963 
15964 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15965 %{
15966   effect( DEF dst, USE src1, USE src2, USE cr );
15967 
15968   ins_cost(INSN_COST * 2);
15969   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15970 
15971   ins_encode %{
15972     __ cselw(as_Register($dst$$reg),
15973              as_Register($src1$$reg),
15974              as_Register($src2$$reg),
15975              Assembler::GT);
15976   %}
15977 
15978   ins_pipe(icond_reg_reg);
15979 %}
15980 
15981 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15982 %{
15983   match(Set dst (MaxI src1 src2));
15984   ins_cost(INSN_COST * 3);
15985   expand %{
15986     rFlagsReg cr;
15987     compI_reg_reg(cr, src1, src2);
15988     cmovI_reg_reg_gt(dst, src1, src2, cr);
15989   %}
15990 %}
15991 
15992 // ============================================================================
15993 // Branch Instructions
15994 
15995 // Direct Branch.
15996 instruct branch(label lbl)
15997 %{
15998   match(Goto);
15999 
16000   effect(USE lbl);
16001 
16002   ins_cost(BRANCH_COST);
16003   format %{ "b  $lbl" %}
16004 
16005   ins_encode(aarch64_enc_b(lbl));
16006 
16007   ins_pipe(pipe_branch);
16008 %}
16009 
16010 // Conditional Near Branch
16011 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16012 %{
16013   // Same match rule as `branchConFar'.
16014   match(If cmp cr);
16015 
16016   effect(USE lbl);
16017 
16018   ins_cost(BRANCH_COST);
16019   // If set to 1 this indicates that the current instruction is a
16020   // short variant of a long branch. This avoids using this
16021   // instruction in first-pass matching. It will then only be used in
16022   // the `Shorten_branches' pass.
16023   // ins_short_branch(1);
16024   format %{ "b$cmp  $lbl" %}
16025 
16026   ins_encode(aarch64_enc_br_con(cmp, lbl));
16027 
16028   ins_pipe(pipe_branch_cond);
16029 %}
16030 
16031 // Conditional Near Branch Unsigned
16032 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16033 %{
16034   // Same match rule as `branchConFar'.
16035   match(If cmp cr);
16036 
16037   effect(USE lbl);
16038 
16039   ins_cost(BRANCH_COST);
16040   // If set to 1 this indicates that the current instruction is a
16041   // short variant of a long branch. This avoids using this
16042   // instruction in first-pass matching. It will then only be used in
16043   // the `Shorten_branches' pass.
16044   // ins_short_branch(1);
16045   format %{ "b$cmp  $lbl\t# unsigned" %}
16046 
16047   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16048 
16049   ins_pipe(pipe_branch_cond);
16050 %}
16051 
16052 // Make use of CBZ and CBNZ.  These instructions, as well as being
16053 // shorter than (cmp; branch), have the additional benefit of not
16054 // killing the flags.
16055 
16056 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16057   match(If cmp (CmpI op1 op2));
16058   effect(USE labl);
16059 
16060   ins_cost(BRANCH_COST);
16061   format %{ "cbw$cmp   $op1, $labl" %}
16062   ins_encode %{
16063     Label* L = $labl$$label;
16064     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16065     if (cond == Assembler::EQ)
16066       __ cbzw($op1$$Register, *L);
16067     else
16068       __ cbnzw($op1$$Register, *L);
16069   %}
16070   ins_pipe(pipe_cmp_branch);
16071 %}
16072 
16073 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16074   match(If cmp (CmpL op1 op2));
16075   effect(USE labl);
16076 
16077   ins_cost(BRANCH_COST);
16078   format %{ "cb$cmp   $op1, $labl" %}
16079   ins_encode %{
16080     Label* L = $labl$$label;
16081     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16082     if (cond == Assembler::EQ)
16083       __ cbz($op1$$Register, *L);
16084     else
16085       __ cbnz($op1$$Register, *L);
16086   %}
16087   ins_pipe(pipe_cmp_branch);
16088 %}
16089 
16090 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16091   match(If cmp (CmpP op1 op2));
16092   effect(USE labl);
16093 
16094   ins_cost(BRANCH_COST);
16095   format %{ "cb$cmp   $op1, $labl" %}
16096   ins_encode %{
16097     Label* L = $labl$$label;
16098     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16099     if (cond == Assembler::EQ)
16100       __ cbz($op1$$Register, *L);
16101     else
16102       __ cbnz($op1$$Register, *L);
16103   %}
16104   ins_pipe(pipe_cmp_branch);
16105 %}
16106 
16107 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16108   match(If cmp (CmpN op1 op2));
16109   effect(USE labl);
16110 
16111   ins_cost(BRANCH_COST);
16112   format %{ "cbw$cmp   $op1, $labl" %}
16113   ins_encode %{
16114     Label* L = $labl$$label;
16115     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16116     if (cond == Assembler::EQ)
16117       __ cbzw($op1$$Register, *L);
16118     else
16119       __ cbnzw($op1$$Register, *L);
16120   %}
16121   ins_pipe(pipe_cmp_branch);
16122 %}
16123 
16124 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16125   match(If cmp (CmpP (DecodeN oop) zero));
16126   effect(USE labl);
16127 
16128   ins_cost(BRANCH_COST);
16129   format %{ "cb$cmp   $oop, $labl" %}
16130   ins_encode %{
16131     Label* L = $labl$$label;
16132     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16133     if (cond == Assembler::EQ)
16134       __ cbzw($oop$$Register, *L);
16135     else
16136       __ cbnzw($oop$$Register, *L);
16137   %}
16138   ins_pipe(pipe_cmp_branch);
16139 %}
16140 
16141 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16142   match(If cmp (CmpU op1 op2));
16143   effect(USE labl);
16144 
16145   ins_cost(BRANCH_COST);
16146   format %{ "cbw$cmp   $op1, $labl" %}
16147   ins_encode %{
16148     Label* L = $labl$$label;
16149     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16150     if (cond == Assembler::EQ || cond == Assembler::LS)
16151       __ cbzw($op1$$Register, *L);
16152     else
16153       __ cbnzw($op1$$Register, *L);
16154   %}
16155   ins_pipe(pipe_cmp_branch);
16156 %}
16157 
16158 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16159   match(If cmp (CmpUL op1 op2));
16160   effect(USE labl);
16161 
16162   ins_cost(BRANCH_COST);
16163   format %{ "cb$cmp   $op1, $labl" %}
16164   ins_encode %{
16165     Label* L = $labl$$label;
16166     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16167     if (cond == Assembler::EQ || cond == Assembler::LS)
16168       __ cbz($op1$$Register, *L);
16169     else
16170       __ cbnz($op1$$Register, *L);
16171   %}
16172   ins_pipe(pipe_cmp_branch);
16173 %}
16174 
16175 // Test bit and Branch
16176 
16177 // Patterns for short (< 32KiB) variants
16178 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16179   match(If cmp (CmpL op1 op2));
16180   effect(USE labl);
16181 
16182   ins_cost(BRANCH_COST);
16183   format %{ "cb$cmp   $op1, $labl # long" %}
16184   ins_encode %{
16185     Label* L = $labl$$label;
16186     Assembler::Condition cond =
16187       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16188     __ tbr(cond, $op1$$Register, 63, *L);
16189   %}
16190   ins_pipe(pipe_cmp_branch);
16191   ins_short_branch(1);
16192 %}
16193 
16194 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16195   match(If cmp (CmpI op1 op2));
16196   effect(USE labl);
16197 
16198   ins_cost(BRANCH_COST);
16199   format %{ "cb$cmp   $op1, $labl # int" %}
16200   ins_encode %{
16201     Label* L = $labl$$label;
16202     Assembler::Condition cond =
16203       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16204     __ tbr(cond, $op1$$Register, 31, *L);
16205   %}
16206   ins_pipe(pipe_cmp_branch);
16207   ins_short_branch(1);
16208 %}
16209 
16210 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16211   match(If cmp (CmpL (AndL op1 op2) op3));
16212   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16213   effect(USE labl);
16214 
16215   ins_cost(BRANCH_COST);
16216   format %{ "tb$cmp   $op1, $op2, $labl" %}
16217   ins_encode %{
16218     Label* L = $labl$$label;
16219     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16220     int bit = exact_log2_long($op2$$constant);
16221     __ tbr(cond, $op1$$Register, bit, *L);
16222   %}
16223   ins_pipe(pipe_cmp_branch);
16224   ins_short_branch(1);
16225 %}
16226 
16227 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16228   match(If cmp (CmpI (AndI op1 op2) op3));
16229   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16230   effect(USE labl);
16231 
16232   ins_cost(BRANCH_COST);
16233   format %{ "tb$cmp   $op1, $op2, $labl" %}
16234   ins_encode %{
16235     Label* L = $labl$$label;
16236     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16237     int bit = exact_log2((juint)$op2$$constant);
16238     __ tbr(cond, $op1$$Register, bit, *L);
16239   %}
16240   ins_pipe(pipe_cmp_branch);
16241   ins_short_branch(1);
16242 %}
16243 
16244 // And far variants
16245 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16246   match(If cmp (CmpL op1 op2));
16247   effect(USE labl);
16248 
16249   ins_cost(BRANCH_COST);
16250   format %{ "cb$cmp   $op1, $labl # long" %}
16251   ins_encode %{
16252     Label* L = $labl$$label;
16253     Assembler::Condition cond =
16254       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16255     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16256   %}
16257   ins_pipe(pipe_cmp_branch);
16258 %}
16259 
16260 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16261   match(If cmp (CmpI op1 op2));
16262   effect(USE labl);
16263 
16264   ins_cost(BRANCH_COST);
16265   format %{ "cb$cmp   $op1, $labl # int" %}
16266   ins_encode %{
16267     Label* L = $labl$$label;
16268     Assembler::Condition cond =
16269       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16270     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16271   %}
16272   ins_pipe(pipe_cmp_branch);
16273 %}
16274 
16275 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16276   match(If cmp (CmpL (AndL op1 op2) op3));
16277   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16278   effect(USE labl);
16279 
16280   ins_cost(BRANCH_COST);
16281   format %{ "tb$cmp   $op1, $op2, $labl" %}
16282   ins_encode %{
16283     Label* L = $labl$$label;
16284     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16285     int bit = exact_log2_long($op2$$constant);
16286     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16287   %}
16288   ins_pipe(pipe_cmp_branch);
16289 %}
16290 
16291 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16292   match(If cmp (CmpI (AndI op1 op2) op3));
16293   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16294   effect(USE labl);
16295 
16296   ins_cost(BRANCH_COST);
16297   format %{ "tb$cmp   $op1, $op2, $labl" %}
16298   ins_encode %{
16299     Label* L = $labl$$label;
16300     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16301     int bit = exact_log2((juint)$op2$$constant);
16302     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16303   %}
16304   ins_pipe(pipe_cmp_branch);
16305 %}
16306 
16307 // Test bits
16308 
16309 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16310   match(Set cr (CmpL (AndL op1 op2) op3));
16311   predicate(Assembler::operand_valid_for_logical_immediate
16312             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16313 
16314   ins_cost(INSN_COST);
16315   format %{ "tst $op1, $op2 # long" %}
16316   ins_encode %{
16317     __ tst($op1$$Register, $op2$$constant);
16318   %}
16319   ins_pipe(ialu_reg_reg);
16320 %}
16321 
16322 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16323   match(Set cr (CmpI (AndI op1 op2) op3));
16324   predicate(Assembler::operand_valid_for_logical_immediate
16325             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16326 
16327   ins_cost(INSN_COST);
16328   format %{ "tst $op1, $op2 # int" %}
16329   ins_encode %{
16330     __ tstw($op1$$Register, $op2$$constant);
16331   %}
16332   ins_pipe(ialu_reg_reg);
16333 %}
16334 
16335 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16336   match(Set cr (CmpL (AndL op1 op2) op3));
16337 
16338   ins_cost(INSN_COST);
16339   format %{ "tst $op1, $op2 # long" %}
16340   ins_encode %{
16341     __ tst($op1$$Register, $op2$$Register);
16342   %}
16343   ins_pipe(ialu_reg_reg);
16344 %}
16345 
16346 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16347   match(Set cr (CmpI (AndI op1 op2) op3));
16348 
16349   ins_cost(INSN_COST);
16350   format %{ "tstw $op1, $op2 # int" %}
16351   ins_encode %{
16352     __ tstw($op1$$Register, $op2$$Register);
16353   %}
16354   ins_pipe(ialu_reg_reg);
16355 %}
16356 
16357 
16358 // Conditional Far Branch
16359 // Conditional Far Branch Unsigned
16360 // TODO: fixme
16361 
16362 // counted loop end branch near
16363 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16364 %{
16365   match(CountedLoopEnd cmp cr);
16366 
16367   effect(USE lbl);
16368 
16369   ins_cost(BRANCH_COST);
16370   // short variant.
16371   // ins_short_branch(1);
16372   format %{ "b$cmp $lbl \t// counted loop end" %}
16373 
16374   ins_encode(aarch64_enc_br_con(cmp, lbl));
16375 
16376   ins_pipe(pipe_branch);
16377 %}
16378 
16379 // counted loop end branch near Unsigned
16380 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16381 %{
16382   match(CountedLoopEnd cmp cr);
16383 
16384   effect(USE lbl);
16385 
16386   ins_cost(BRANCH_COST);
16387   // short variant.
16388   // ins_short_branch(1);
16389   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16390 
16391   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16392 
16393   ins_pipe(pipe_branch);
16394 %}
16395 
16396 // counted loop end branch far
16397 // counted loop end branch far unsigned
16398 // TODO: fixme
16399 
16400 // ============================================================================
16401 // inlined locking and unlocking
16402 
16403 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16404 %{
16405   match(Set cr (FastLock object box));
16406   effect(TEMP tmp, TEMP tmp2);
16407 
16408   // TODO
16409   // identify correct cost
16410   ins_cost(5 * INSN_COST);
16411   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16412 
16413   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16414 
16415   ins_pipe(pipe_serial);
16416 %}
16417 
16418 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16419 %{
16420   match(Set cr (FastUnlock object box));
16421   effect(TEMP tmp, TEMP tmp2);
16422 
16423   ins_cost(5 * INSN_COST);
16424   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16425 
16426   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16427 
16428   ins_pipe(pipe_serial);
16429 %}
16430 
16431 
16432 // ============================================================================
16433 // Safepoint Instructions
16434 
16435 // TODO
16436 // provide a near and far version of this code
16437 
16438 instruct safePoint(rFlagsReg cr, iRegP poll)
16439 %{
16440   match(SafePoint poll);
16441   effect(KILL cr);
16442 
16443   format %{
16444     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16445   %}
16446   ins_encode %{
16447     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16448   %}
16449   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16450 %}
16451 
16452 
16453 // ============================================================================
16454 // Procedure Call/Return Instructions
16455 
16456 // Call Java Static Instruction
16457 
16458 instruct CallStaticJavaDirect(method meth)
16459 %{
16460   match(CallStaticJava);
16461 
16462   effect(USE meth);
16463 
16464   ins_cost(CALL_COST);
16465 
16466   format %{ "call,static $meth \t// ==> " %}
16467 
16468   ins_encode(aarch64_enc_java_static_call(meth),
16469              aarch64_enc_call_epilog);
16470 
16471   ins_pipe(pipe_class_call);
16472 %}
16473 
16474 // TO HERE
16475 
16476 // Call Java Dynamic Instruction
16477 instruct CallDynamicJavaDirect(method meth)
16478 %{
16479   match(CallDynamicJava);
16480 
16481   effect(USE meth);
16482 
16483   ins_cost(CALL_COST);
16484 
16485   format %{ "CALL,dynamic $meth \t// ==> " %}
16486 
16487   ins_encode(aarch64_enc_java_dynamic_call(meth),
16488              aarch64_enc_call_epilog);
16489 
16490   ins_pipe(pipe_class_call);
16491 %}
16492 
16493 // Call Runtime Instruction
16494 
16495 instruct CallRuntimeDirect(method meth)
16496 %{
16497   match(CallRuntime);
16498 
16499   effect(USE meth);
16500 
16501   ins_cost(CALL_COST);
16502 
16503   format %{ "CALL, runtime $meth" %}
16504 
16505   ins_encode( aarch64_enc_java_to_runtime(meth) );
16506 
16507   ins_pipe(pipe_class_call);
16508 %}
16509 
16510 // Call Runtime Instruction
16511 
16512 instruct CallLeafDirect(method meth)
16513 %{
16514   match(CallLeaf);
16515 
16516   effect(USE meth);
16517 
16518   ins_cost(CALL_COST);
16519 
16520   format %{ "CALL, runtime leaf $meth" %}
16521 
16522   ins_encode( aarch64_enc_java_to_runtime(meth) );
16523 
16524   ins_pipe(pipe_class_call);
16525 %}
16526 
16527 // Call Runtime Instruction
16528 
16529 instruct CallLeafNoFPDirect(method meth)
16530 %{
16531   match(CallLeafNoFP);
16532 
16533   effect(USE meth);
16534 
16535   ins_cost(CALL_COST);
16536 
16537   format %{ "CALL, runtime leaf nofp $meth" %}
16538 
16539   ins_encode( aarch64_enc_java_to_runtime(meth) );
16540 
16541   ins_pipe(pipe_class_call);
16542 %}
16543 
16544 instruct CallNativeDirect(method meth)
16545 %{
16546   match(CallNative);
16547 
16548   effect(USE meth);
16549 
16550   ins_cost(CALL_COST);
16551 
16552   format %{ "CALL, native $meth" %}
16553 
16554   ins_encode( aarch64_enc_java_to_runtime(meth) );
16555 
16556   ins_pipe(pipe_class_call);
16557 %}
16558 
16559 // Tail Call; Jump from runtime stub to Java code.
16560 // Also known as an 'interprocedural jump'.
16561 // Target of jump will eventually return to caller.
16562 // TailJump below removes the return address.
16563 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16564 %{
16565   match(TailCall jump_target method_ptr);
16566 
16567   ins_cost(CALL_COST);
16568 
16569   format %{ "br $jump_target\t# $method_ptr holds method" %}
16570 
16571   ins_encode(aarch64_enc_tail_call(jump_target));
16572 
16573   ins_pipe(pipe_class_call);
16574 %}
16575 
16576 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16577 %{
16578   match(TailJump jump_target ex_oop);
16579 
16580   ins_cost(CALL_COST);
16581 
16582   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16583 
16584   ins_encode(aarch64_enc_tail_jmp(jump_target));
16585 
16586   ins_pipe(pipe_class_call);
16587 %}
16588 
16589 // Create exception oop: created by stack-crawling runtime code.
16590 // Created exception is now available to this handler, and is setup
16591 // just prior to jumping to this handler. No code emitted.
16592 // TODO check
16593 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16594 instruct CreateException(iRegP_R0 ex_oop)
16595 %{
16596   match(Set ex_oop (CreateEx));
16597 
16598   format %{ " -- \t// exception oop; no code emitted" %}
16599 
16600   size(0);
16601 
16602   ins_encode( /*empty*/ );
16603 
16604   ins_pipe(pipe_class_empty);
16605 %}
16606 
16607 // Rethrow exception: The exception oop will come in the first
16608 // argument position. Then JUMP (not call) to the rethrow stub code.
16609 instruct RethrowException() %{
16610   match(Rethrow);
16611   ins_cost(CALL_COST);
16612 
16613   format %{ "b rethrow_stub" %}
16614 
16615   ins_encode( aarch64_enc_rethrow() );
16616 
16617   ins_pipe(pipe_class_call);
16618 %}
16619 
16620 
16621 // Return Instruction
16622 // epilog node loads ret address into lr as part of frame pop
16623 instruct Ret()
16624 %{
16625   match(Return);
16626 
16627   format %{ "ret\t// return register" %}
16628 
16629   ins_encode( aarch64_enc_ret() );
16630 
16631   ins_pipe(pipe_branch);
16632 %}
16633 
16634 // Die now.
16635 instruct ShouldNotReachHere() %{
16636   match(Halt);
16637 
16638   ins_cost(CALL_COST);
16639   format %{ "ShouldNotReachHere" %}
16640 
16641   ins_encode %{
16642     if (is_reachable()) {
16643       __ stop(_halt_reason);
16644     }
16645   %}
16646 
16647   ins_pipe(pipe_class_default);
16648 %}
16649 
16650 // ============================================================================
16651 // Partial Subtype Check
16652 //
16653 // superklass array for an instance of the superklass.  Set a hidden
16654 // internal cache on a hit (cache is checked with exposed code in
16655 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16656 // encoding ALSO sets flags.
16657 
16658 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16659 %{
16660   match(Set result (PartialSubtypeCheck sub super));
16661   effect(KILL cr, KILL temp);
16662 
16663   ins_cost(1100);  // slightly larger than the next version
16664   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16665 
16666   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16667 
16668   opcode(0x1); // Force zero of result reg on hit
16669 
16670   ins_pipe(pipe_class_memory);
16671 %}
16672 
16673 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16674 %{
16675   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16676   effect(KILL temp, KILL result);
16677 
16678   ins_cost(1100);  // slightly larger than the next version
16679   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16680 
16681   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16682 
16683   opcode(0x0); // Don't zero result reg on hit
16684 
16685   ins_pipe(pipe_class_memory);
16686 %}
16687 
16688 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16689                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16690 %{
16691   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16692   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16693   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16694 
16695   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16696   ins_encode %{
16697     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16698     __ string_compare($str1$$Register, $str2$$Register,
16699                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16700                       $tmp1$$Register, $tmp2$$Register,
16701                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16702   %}
16703   ins_pipe(pipe_class_memory);
16704 %}
16705 
16706 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16707                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16708 %{
16709   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16710   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16711   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16712 
16713   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16714   ins_encode %{
16715     __ string_compare($str1$$Register, $str2$$Register,
16716                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16717                       $tmp1$$Register, $tmp2$$Register,
16718                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16719   %}
16720   ins_pipe(pipe_class_memory);
16721 %}
16722 
16723 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16724                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16725                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16726 %{
16727   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16728   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16729   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16730          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16731 
16732   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16733   ins_encode %{
16734     __ string_compare($str1$$Register, $str2$$Register,
16735                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16736                       $tmp1$$Register, $tmp2$$Register,
16737                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16738                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16739   %}
16740   ins_pipe(pipe_class_memory);
16741 %}
16742 
16743 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16744                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16745                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16746 %{
16747   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16748   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16749   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16750          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16751 
16752   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16753   ins_encode %{
16754     __ string_compare($str1$$Register, $str2$$Register,
16755                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16756                       $tmp1$$Register, $tmp2$$Register,
16757                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16758                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16759   %}
16760   ins_pipe(pipe_class_memory);
16761 %}
16762 
16763 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16764        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16765        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16766 %{
16767   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16768   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16769   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16770          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16771   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16772 
16773   ins_encode %{
16774     __ string_indexof($str1$$Register, $str2$$Register,
16775                       $cnt1$$Register, $cnt2$$Register,
16776                       $tmp1$$Register, $tmp2$$Register,
16777                       $tmp3$$Register, $tmp4$$Register,
16778                       $tmp5$$Register, $tmp6$$Register,
16779                       -1, $result$$Register, StrIntrinsicNode::UU);
16780   %}
16781   ins_pipe(pipe_class_memory);
16782 %}
16783 
16784 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16785        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16786        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16787 %{
16788   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16789   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16790   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16791          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16792   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16793 
16794   ins_encode %{
16795     __ string_indexof($str1$$Register, $str2$$Register,
16796                       $cnt1$$Register, $cnt2$$Register,
16797                       $tmp1$$Register, $tmp2$$Register,
16798                       $tmp3$$Register, $tmp4$$Register,
16799                       $tmp5$$Register, $tmp6$$Register,
16800                       -1, $result$$Register, StrIntrinsicNode::LL);
16801   %}
16802   ins_pipe(pipe_class_memory);
16803 %}
16804 
16805 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16806        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16807        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16808 %{
16809   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16810   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16811   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16812          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16813   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16814 
16815   ins_encode %{
16816     __ string_indexof($str1$$Register, $str2$$Register,
16817                       $cnt1$$Register, $cnt2$$Register,
16818                       $tmp1$$Register, $tmp2$$Register,
16819                       $tmp3$$Register, $tmp4$$Register,
16820                       $tmp5$$Register, $tmp6$$Register,
16821                       -1, $result$$Register, StrIntrinsicNode::UL);
16822   %}
16823   ins_pipe(pipe_class_memory);
16824 %}
16825 
16826 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16827                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16828                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16829 %{
16830   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16831   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16832   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16833          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16834   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16835 
16836   ins_encode %{
16837     int icnt2 = (int)$int_cnt2$$constant;
16838     __ string_indexof($str1$$Register, $str2$$Register,
16839                       $cnt1$$Register, zr,
16840                       $tmp1$$Register, $tmp2$$Register,
16841                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16842                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16843   %}
16844   ins_pipe(pipe_class_memory);
16845 %}
16846 
16847 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16848                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16849                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16850 %{
16851   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16852   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16853   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16854          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16855   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16856 
16857   ins_encode %{
16858     int icnt2 = (int)$int_cnt2$$constant;
16859     __ string_indexof($str1$$Register, $str2$$Register,
16860                       $cnt1$$Register, zr,
16861                       $tmp1$$Register, $tmp2$$Register,
16862                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16863                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16864   %}
16865   ins_pipe(pipe_class_memory);
16866 %}
16867 
16868 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16869                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16870                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16871 %{
16872   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16873   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16874   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16875          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16876   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16877 
16878   ins_encode %{
16879     int icnt2 = (int)$int_cnt2$$constant;
16880     __ string_indexof($str1$$Register, $str2$$Register,
16881                       $cnt1$$Register, zr,
16882                       $tmp1$$Register, $tmp2$$Register,
16883                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16884                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16885   %}
16886   ins_pipe(pipe_class_memory);
16887 %}
16888 
16889 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16890                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16891                              iRegINoSp tmp3, rFlagsReg cr)
16892 %{
16893   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16894   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16895   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16896          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16897 
16898   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16899 
16900   ins_encode %{
16901     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16902                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16903                            $tmp3$$Register);
16904   %}
16905   ins_pipe(pipe_class_memory);
16906 %}
16907 
16908 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16909                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16910                               iRegINoSp tmp3, rFlagsReg cr)
16911 %{
16912   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16913   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16914   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16915          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16916 
16917   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16918 
16919   ins_encode %{
16920     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16921                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16922                             $tmp3$$Register);
16923   %}
16924   ins_pipe(pipe_class_memory);
16925 %}
16926 
16927 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16928                         iRegI_R0 result, rFlagsReg cr)
16929 %{
16930   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16931   match(Set result (StrEquals (Binary str1 str2) cnt));
16932   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16933 
16934   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16935   ins_encode %{
16936     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16937     __ string_equals($str1$$Register, $str2$$Register,
16938                      $result$$Register, $cnt$$Register, 1);
16939   %}
16940   ins_pipe(pipe_class_memory);
16941 %}
16942 
16943 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16944                         iRegI_R0 result, rFlagsReg cr)
16945 %{
16946   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16947   match(Set result (StrEquals (Binary str1 str2) cnt));
16948   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16949 
16950   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16951   ins_encode %{
16952     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16953     __ string_equals($str1$$Register, $str2$$Register,
16954                      $result$$Register, $cnt$$Register, 2);
16955   %}
16956   ins_pipe(pipe_class_memory);
16957 %}
16958 
16959 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16960                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16961                        iRegP_R10 tmp, rFlagsReg cr)
16962 %{
16963   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16964   match(Set result (AryEq ary1 ary2));
16965   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16966 
16967   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16968   ins_encode %{
16969     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16970                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16971                                    $result$$Register, $tmp$$Register, 1);
16972     if (tpc == NULL) {
16973       ciEnv::current()->record_failure("CodeCache is full");
16974       return;
16975     }
16976   %}
16977   ins_pipe(pipe_class_memory);
16978 %}
16979 
16980 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16981                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16982                        iRegP_R10 tmp, rFlagsReg cr)
16983 %{
16984   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16985   match(Set result (AryEq ary1 ary2));
16986   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16987 
16988   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16989   ins_encode %{
16990     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16991                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16992                                    $result$$Register, $tmp$$Register, 2);
16993     if (tpc == NULL) {
16994       ciEnv::current()->record_failure("CodeCache is full");
16995       return;
16996     }
16997   %}
16998   ins_pipe(pipe_class_memory);
16999 %}
17000 
17001 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17002 %{
17003   match(Set result (HasNegatives ary1 len));
17004   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17005   format %{ "has negatives byte[] $ary1,$len -> $result" %}
17006   ins_encode %{
17007     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17008     if (tpc == NULL) {
17009       ciEnv::current()->record_failure("CodeCache is full");
17010       return;
17011     }
17012   %}
17013   ins_pipe( pipe_slow );
17014 %}
17015 
17016 // fast char[] to byte[] compression
17017 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17018                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17019                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17020                          iRegI_R0 result, rFlagsReg cr)
17021 %{
17022   match(Set result (StrCompressedCopy src (Binary dst len)));
17023   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17024 
17025   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
17026   ins_encode %{
17027     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17028                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17029                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17030                            $result$$Register);
17031   %}
17032   ins_pipe( pipe_slow );
17033 %}
17034 
17035 // fast byte[] to char[] inflation
17036 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17037                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17038 %{
17039   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17040   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17041 
17042   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17043   ins_encode %{
17044     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17045                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17046                                         $tmp3$$FloatRegister, $tmp4$$Register);
17047     if (tpc == NULL) {
17048       ciEnv::current()->record_failure("CodeCache is full");
17049       return;
17050     }
17051   %}
17052   ins_pipe(pipe_class_memory);
17053 %}
17054 
17055 // encode char[] to byte[] in ISO_8859_1
17056 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17057                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17058                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17059                           iRegI_R0 result, rFlagsReg cr)
17060 %{
17061   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17062   match(Set result (EncodeISOArray src (Binary dst len)));
17063   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17064          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17065 
17066   format %{ "Encode array $src,$dst,$len -> $result" %}
17067   ins_encode %{
17068     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17069          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
17070          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
17071   %}
17072   ins_pipe( pipe_class_memory );
17073 %}
17074 
17075 // ============================================================================
17076 // This name is KNOWN by the ADLC and cannot be changed.
17077 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17078 // for this guy.
17079 instruct tlsLoadP(thread_RegP dst)
17080 %{
17081   match(Set dst (ThreadLocal));
17082 
17083   ins_cost(0);
17084 
17085   format %{ " -- \t// $dst=Thread::current(), empty" %}
17086 
17087   size(0);
17088 
17089   ins_encode( /*empty*/ );
17090 
17091   ins_pipe(pipe_class_empty);
17092 %}
17093 
17094 //----------PEEPHOLE RULES-----------------------------------------------------
17095 // These must follow all instruction definitions as they use the names
17096 // defined in the instructions definitions.
17097 //
17098 // peepmatch ( root_instr_name [preceding_instruction]* );
17099 //
17100 // peepconstraint %{
17101 // (instruction_number.operand_name relational_op instruction_number.operand_name
17102 //  [, ...] );
17103 // // instruction numbers are zero-based using left to right order in peepmatch
17104 //
17105 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17106 // // provide an instruction_number.operand_name for each operand that appears
17107 // // in the replacement instruction's match rule
17108 //
17109 // ---------VM FLAGS---------------------------------------------------------
17110 //
17111 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17112 //
17113 // Each peephole rule is given an identifying number starting with zero and
17114 // increasing by one in the order seen by the parser.  An individual peephole
17115 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17116 // on the command-line.
17117 //
17118 // ---------CURRENT LIMITATIONS----------------------------------------------
17119 //
17120 // Only match adjacent instructions in same basic block
17121 // Only equality constraints
17122 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17123 // Only one replacement instruction
17124 //
17125 // ---------EXAMPLE----------------------------------------------------------
17126 //
17127 // // pertinent parts of existing instructions in architecture description
17128 // instruct movI(iRegINoSp dst, iRegI src)
17129 // %{
17130 //   match(Set dst (CopyI src));
17131 // %}
17132 //
17133 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17134 // %{
17135 //   match(Set dst (AddI dst src));
17136 //   effect(KILL cr);
17137 // %}
17138 //
17139 // // Change (inc mov) to lea
17140 // peephole %{
17141 //   // increment preceeded by register-register move
17142 //   peepmatch ( incI_iReg movI );
17143 //   // require that the destination register of the increment
17144 //   // match the destination register of the move
17145 //   peepconstraint ( 0.dst == 1.dst );
17146 //   // construct a replacement instruction that sets
17147 //   // the destination to ( move's source register + one )
17148 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17149 // %}
17150 //
17151 
17152 // Implementation no longer uses movX instructions since
17153 // machine-independent system no longer uses CopyX nodes.
17154 //
17155 // peephole
17156 // %{
17157 //   peepmatch (incI_iReg movI);
17158 //   peepconstraint (0.dst == 1.dst);
17159 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17160 // %}
17161 
17162 // peephole
17163 // %{
17164 //   peepmatch (decI_iReg movI);
17165 //   peepconstraint (0.dst == 1.dst);
17166 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17167 // %}
17168 
17169 // peephole
17170 // %{
17171 //   peepmatch (addI_iReg_imm movI);
17172 //   peepconstraint (0.dst == 1.dst);
17173 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17174 // %}
17175 
17176 // peephole
17177 // %{
17178 //   peepmatch (incL_iReg movL);
17179 //   peepconstraint (0.dst == 1.dst);
17180 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17181 // %}
17182 
17183 // peephole
17184 // %{
17185 //   peepmatch (decL_iReg movL);
17186 //   peepconstraint (0.dst == 1.dst);
17187 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17188 // %}
17189 
17190 // peephole
17191 // %{
17192 //   peepmatch (addL_iReg_imm movL);
17193 //   peepconstraint (0.dst == 1.dst);
17194 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17195 // %}
17196 
17197 // peephole
17198 // %{
17199 //   peepmatch (addP_iReg_imm movP);
17200 //   peepconstraint (0.dst == 1.dst);
17201 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17202 // %}
17203 
17204 // // Change load of spilled value to only a spill
17205 // instruct storeI(memory mem, iRegI src)
17206 // %{
17207 //   match(Set mem (StoreI mem src));
17208 // %}
17209 //
17210 // instruct loadI(iRegINoSp dst, memory mem)
17211 // %{
17212 //   match(Set dst (LoadI mem));
17213 // %}
17214 //
17215 
17216 //----------SMARTSPILL RULES---------------------------------------------------
17217 // These must follow all instruction definitions as they use the names
17218 // defined in the instructions definitions.
17219 
17220 // Local Variables:
17221 // mode: c++
17222 // End: