1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return 4 * NativeInstruction::instruction_size;
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298   bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else if (_entry_point == NULL) {
 1771     // See CallLeafNoFPIndirect
 1772     return 1 * NativeInstruction::instruction_size;
 1773   } else {
 1774     return 6 * NativeInstruction::instruction_size;
 1775   }
 1776 }
 1777 
 1778 int MachCallNativeNode::ret_addr_offset() {
 1779   // This is implemented using aarch64_enc_java_to_runtime as above.
 1780   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1781   if (cb) {
 1782     return 1 * NativeInstruction::instruction_size;
 1783   } else {
 1784     return 6 * NativeInstruction::instruction_size;
 1785   }
 1786 }
 1787 
 1788 //=============================================================================
 1789 
 1790 #ifndef PRODUCT
 1791 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1792   st->print("BREAKPOINT");
 1793 }
 1794 #endif
 1795 
 1796 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1797   C2_MacroAssembler _masm(&cbuf);
 1798   __ brk(0);
 1799 }
 1800 
 1801 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1802   return MachNode::size(ra_);
 1803 }
 1804 
 1805 //=============================================================================
 1806 
 1807 #ifndef PRODUCT
 1808   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1809     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1810   }
 1811 #endif
 1812 
 1813   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1814     C2_MacroAssembler _masm(&cbuf);
 1815     for (int i = 0; i < _count; i++) {
 1816       __ nop();
 1817     }
 1818   }
 1819 
 1820   uint MachNopNode::size(PhaseRegAlloc*) const {
 1821     return _count * NativeInstruction::instruction_size;
 1822   }
 1823 
 1824 //=============================================================================
 1825 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1826 
 1827 int ConstantTable::calculate_table_base_offset() const {
 1828   return 0;  // absolute addressing, no offset
 1829 }
 1830 
 1831 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1832 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1833   ShouldNotReachHere();
 1834 }
 1835 
 1836 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1837   // Empty encoding
 1838 }
 1839 
 1840 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1841   return 0;
 1842 }
 1843 
 1844 #ifndef PRODUCT
 1845 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1846   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1847 }
 1848 #endif
 1849 
 1850 #ifndef PRODUCT
 1851 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1852   Compile* C = ra_->C;
 1853 
 1854   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1855 
 1856   if (C->output()->need_stack_bang(framesize))
 1857     st->print("# stack bang size=%d\n\t", framesize);
 1858 
 1859   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1860     st->print("sub  sp, sp, #%d\n\t", framesize);
 1861     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1862     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1863   } else {
 1864     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1865     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1866     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1867     st->print("sub  sp, sp, rscratch1");
 1868   }
 1869   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1870     st->print("\n\t");
 1871     st->print("ldr  rscratch1, [guard]\n\t");
 1872     st->print("dmb ishld\n\t");
 1873     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1874     st->print("cmp  rscratch1, rscratch2\n\t");
 1875     st->print("b.eq skip");
 1876     st->print("\n\t");
 1877     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1878     st->print("b skip\n\t");
 1879     st->print("guard: int\n\t");
 1880     st->print("\n\t");
 1881     st->print("skip:\n\t");
 1882   }
 1883 }
 1884 #endif
 1885 
 1886 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1887   Compile* C = ra_->C;
 1888   C2_MacroAssembler _masm(&cbuf);
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() > 0) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   __ verified_entry(C, 0);
 1910   __ bind(*_verified_entry);
 1911 
 1912   if (C->stub_function() == NULL) {
 1913     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1914     bs->nmethod_entry_barrier(&_masm);
 1915   }
 1916 
 1917   if (VerifyStackAtCalls) {
 1918     Unimplemented();
 1919   }
 1920 
 1921   C->output()->set_frame_complete(cbuf.insts_size());
 1922 
 1923   if (C->has_mach_constant_base_node()) {
 1924     // NOTE: We set the table base offset here because users might be
 1925     // emitted before MachConstantBaseNode.
 1926     ConstantTable& constant_table = C->output()->constant_table();
 1927     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1928   }
 1929 }
 1930 
 1931 int MachPrologNode::reloc() const
 1932 {
 1933   return 0;
 1934 }
 1935 
 1936 //=============================================================================
 1937 
 1938 #ifndef PRODUCT
 1939 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1940   Compile* C = ra_->C;
 1941   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1942 
 1943   st->print("# pop frame %d\n\t",framesize);
 1944 
 1945   if (framesize == 0) {
 1946     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1947   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1948     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1949     st->print("add  sp, sp, #%d\n\t", framesize);
 1950   } else {
 1951     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1952     st->print("add  sp, sp, rscratch1\n\t");
 1953     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1954   }
 1955 
 1956   if (do_polling() && C->is_method_compilation()) {
 1957     st->print("# test polling word\n\t");
 1958     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1959     st->print("cmp  sp, rscratch1\n\t");
 1960     st->print("bhi #slow_path");
 1961   }
 1962 }
 1963 #endif
 1964 
 1965 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1966   Compile* C = ra_->C;
 1967   C2_MacroAssembler _masm(&cbuf);
 1968   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1969 
 1970   __ remove_frame(framesize, C->needs_stack_repair());
 1971 
 1972   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1973     __ reserved_stack_check();
 1974   }
 1975 
 1976   if (do_polling() && C->is_method_compilation()) {
 1977     Label dummy_label;
 1978     Label* code_stub = &dummy_label;
 1979     if (!C->output()->in_scratch_emit_size()) {
 1980       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1981     }
 1982     __ relocate(relocInfo::poll_return_type);
 1983     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1984   }
 1985 }
 1986 
 1987 int MachEpilogNode::reloc() const {
 1988   // Return number of relocatable values contained in this instruction.
 1989   return 1; // 1 for polling page.
 1990 }
 1991 
 1992 const Pipeline * MachEpilogNode::pipeline() const {
 1993   return MachNode::pipeline_class();
 1994 }
 1995 
 1996 //=============================================================================
 1997 
 1998 // Figure out which register class each belongs in: rc_int, rc_float or
 1999 // rc_stack.
 2000 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2001 
 2002 static enum RC rc_class(OptoReg::Name reg) {
 2003 
 2004   if (reg == OptoReg::Bad) {
 2005     return rc_bad;
 2006   }
 2007 
 2008   // we have 32 int registers * 2 halves
 2009   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2010 
 2011   if (reg < slots_of_int_registers) {
 2012     return rc_int;
 2013   }
 2014 
 2015   // we have 32 float register * 8 halves
 2016   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2017   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2018     return rc_float;
 2019   }
 2020 
 2021   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2022   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2023     return rc_predicate;
 2024   }
 2025 
 2026   // Between predicate regs & stack is the flags.
 2027   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2028 
 2029   return rc_stack;
 2030 }
 2031 
 2032 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2033   Compile* C = ra_->C;
 2034 
 2035   // Get registers to move.
 2036   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2037   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2038   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2039   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2040 
 2041   enum RC src_hi_rc = rc_class(src_hi);
 2042   enum RC src_lo_rc = rc_class(src_lo);
 2043   enum RC dst_hi_rc = rc_class(dst_hi);
 2044   enum RC dst_lo_rc = rc_class(dst_lo);
 2045 
 2046   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2047 
 2048   if (src_hi != OptoReg::Bad) {
 2049     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2050            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2051            "expected aligned-adjacent pairs");
 2052   }
 2053 
 2054   if (src_lo == dst_lo && src_hi == dst_hi) {
 2055     return 0;            // Self copy, no move.
 2056   }
 2057 
 2058   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2059               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2060   int src_offset = ra_->reg2offset(src_lo);
 2061   int dst_offset = ra_->reg2offset(dst_lo);
 2062 
 2063   if (bottom_type()->isa_vect() != NULL) {
 2064     uint ireg = ideal_reg();
 2065     if (ireg == Op_VecA && cbuf) {
 2066       C2_MacroAssembler _masm(cbuf);
 2067       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2068       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2069         // stack->stack
 2070         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2071                                                 sve_vector_reg_size_in_bytes);
 2072       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2073         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2074                             sve_vector_reg_size_in_bytes);
 2075       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2076         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2077                               sve_vector_reg_size_in_bytes);
 2078       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2079         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2080                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2081                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2082       } else {
 2083         ShouldNotReachHere();
 2084       }
 2085     } else if (cbuf) {
 2086       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2087       C2_MacroAssembler _masm(cbuf);
 2088       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2089       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2090         // stack->stack
 2091         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2092         if (ireg == Op_VecD) {
 2093           __ unspill(rscratch1, true, src_offset);
 2094           __ spill(rscratch1, true, dst_offset);
 2095         } else {
 2096           __ spill_copy128(src_offset, dst_offset);
 2097         }
 2098       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2099         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2100                ireg == Op_VecD ? __ T8B : __ T16B,
 2101                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2102       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2103         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2104                  ireg == Op_VecD ? __ D : __ Q,
 2105                  ra_->reg2offset(dst_lo));
 2106       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2107         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2108                    ireg == Op_VecD ? __ D : __ Q,
 2109                    ra_->reg2offset(src_lo));
 2110       } else {
 2111         ShouldNotReachHere();
 2112       }
 2113     }
 2114   } else if (cbuf) {
 2115     C2_MacroAssembler _masm(cbuf);
 2116     switch (src_lo_rc) {
 2117     case rc_int:
 2118       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2119         if (is64) {
 2120             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2121                    as_Register(Matcher::_regEncode[src_lo]));
 2122         } else {
 2123             C2_MacroAssembler _masm(cbuf);
 2124             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2125                     as_Register(Matcher::_regEncode[src_lo]));
 2126         }
 2127       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2128         if (is64) {
 2129             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2130                      as_Register(Matcher::_regEncode[src_lo]));
 2131         } else {
 2132             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2133                      as_Register(Matcher::_regEncode[src_lo]));
 2134         }
 2135       } else {                    // gpr --> stack spill
 2136         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2137         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2138       }
 2139       break;
 2140     case rc_float:
 2141       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2142         if (is64) {
 2143             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2144                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2145         } else {
 2146             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2147                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2148         }
 2149       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2150         if (is64) {
 2151             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2152                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2153         } else {
 2154             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2155                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2156         }
 2157       } else {                    // fpr --> stack spill
 2158         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2159         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2160                  is64 ? __ D : __ S, dst_offset);
 2161       }
 2162       break;
 2163     case rc_stack:
 2164       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2165         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2166       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2167         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2168                    is64 ? __ D : __ S, src_offset);
 2169       } else {                    // stack --> stack copy
 2170         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2171         __ unspill(rscratch1, is64, src_offset);
 2172         __ spill(rscratch1, is64, dst_offset);
 2173       }
 2174       break;
 2175     default:
 2176       assert(false, "bad rc_class for spill");
 2177       ShouldNotReachHere();
 2178     }
 2179   }
 2180 
 2181   if (st) {
 2182     st->print("spill ");
 2183     if (src_lo_rc == rc_stack) {
 2184       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2185     } else {
 2186       st->print("%s -> ", Matcher::regName[src_lo]);
 2187     }
 2188     if (dst_lo_rc == rc_stack) {
 2189       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2190     } else {
 2191       st->print("%s", Matcher::regName[dst_lo]);
 2192     }
 2193     if (bottom_type()->isa_vect() != NULL) {
 2194       int vsize = 0;
 2195       switch (ideal_reg()) {
 2196       case Op_VecD:
 2197         vsize = 64;
 2198         break;
 2199       case Op_VecX:
 2200         vsize = 128;
 2201         break;
 2202       case Op_VecA:
 2203         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2204         break;
 2205       default:
 2206         assert(false, "bad register type for spill");
 2207         ShouldNotReachHere();
 2208       }
 2209       st->print("\t# vector spill size = %d", vsize);
 2210     } else {
 2211       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2212     }
 2213   }
 2214 
 2215   return 0;
 2216 
 2217 }
 2218 
 2219 #ifndef PRODUCT
 2220 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2221   if (!ra_)
 2222     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2223   else
 2224     implementation(NULL, ra_, false, st);
 2225 }
 2226 #endif
 2227 
 2228 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2229   implementation(&cbuf, ra_, false, NULL);
 2230 }
 2231 
 2232 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2233   return MachNode::size(ra_);
 2234 }
 2235 
 2236 //=============================================================================
 2237 
 2238 #ifndef PRODUCT
 2239 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2240   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2241   int reg = ra_->get_reg_first(this);
 2242   st->print("add %s, rsp, #%d]\t# box lock",
 2243             Matcher::regName[reg], offset);
 2244 }
 2245 #endif
 2246 
 2247 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2248   C2_MacroAssembler _masm(&cbuf);
 2249 
 2250   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2251   int reg    = ra_->get_encode(this);
 2252 
 2253   // This add will handle any 24-bit signed offset. 24 bits allows an
 2254   // 8 megabyte stack frame.
 2255   __ add(as_Register(reg), sp, offset);
 2256 }
 2257 
 2258 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2259   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2260   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2261 
 2262   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2263     return NativeInstruction::instruction_size;
 2264   } else {
 2265     return 2 * NativeInstruction::instruction_size;
 2266   }
 2267 }
 2268 
 2269 ///=============================================================================
 2270 #ifndef PRODUCT
 2271 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2272 {
 2273   st->print_cr("# MachVEPNode");
 2274   if (!_verified) {
 2275     st->print_cr("\t load_class");
 2276   } else {
 2277     st->print_cr("\t unpack_inline_arg");
 2278   }
 2279 }
 2280 #endif
 2281 
 2282 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2283 {
 2284   MacroAssembler _masm(&cbuf);
 2285 
 2286   if (!_verified) {
 2287     Label skip;
 2288     __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2289     __ br(Assembler::EQ, skip);
 2290       __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2291     __ bind(skip);
 2292 
 2293   } else {
 2294     // Unpack inline type args passed as oop and then jump to
 2295     // the verified entry point (skipping the unverified entry).
 2296     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2297     // Emit code for verified entry and save increment for stack repair on return
 2298     __ verified_entry(ra_->C, sp_inc);
 2299     __ b(*_verified_entry);
 2300   }
 2301 }
 2302 
 2303 //=============================================================================
 2304 #ifndef PRODUCT
 2305 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2306 {
 2307   st->print_cr("# MachUEPNode");
 2308   if (UseCompressedClassPointers) {
 2309     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2310     if (CompressedKlassPointers::shift() != 0) {
 2311       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2312     }
 2313   } else {
 2314    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2315   }
 2316   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2317   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2318 }
 2319 #endif
 2320 
 2321 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2322 {
 2323   // This is the unverified entry point.
 2324   C2_MacroAssembler _masm(&cbuf);
 2325   Label skip;
 2326 
 2327   // UseCompressedClassPointers logic are inside cmp_klass
 2328   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2329 
 2330   // TODO
 2331   // can we avoid this skip and still use a reloc?
 2332   __ br(Assembler::EQ, skip);
 2333   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2334   __ bind(skip);
 2335 }
 2336 
 2337 // REQUIRED EMIT CODE
 2338 
 2339 //=============================================================================
 2340 
 2341 // Emit exception handler code.
 2342 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2343 {
 2344   // mov rscratch1 #exception_blob_entry_point
 2345   // br rscratch1
 2346   // Note that the code buffer's insts_mark is always relative to insts.
 2347   // That's why we must use the macroassembler to generate a handler.
 2348   C2_MacroAssembler _masm(&cbuf);
 2349   address base = __ start_a_stub(size_exception_handler());
 2350   if (base == NULL) {
 2351     ciEnv::current()->record_failure("CodeCache is full");
 2352     return 0;  // CodeBuffer::expand failed
 2353   }
 2354   int offset = __ offset();
 2355   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2356   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2357   __ end_a_stub();
 2358   return offset;
 2359 }
 2360 
 2361 // Emit deopt handler code.
 2362 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2363 {
 2364   // Note that the code buffer's insts_mark is always relative to insts.
 2365   // That's why we must use the macroassembler to generate a handler.
 2366   C2_MacroAssembler _masm(&cbuf);
 2367   address base = __ start_a_stub(size_deopt_handler());
 2368   if (base == NULL) {
 2369     ciEnv::current()->record_failure("CodeCache is full");
 2370     return 0;  // CodeBuffer::expand failed
 2371   }
 2372   int offset = __ offset();
 2373 
 2374   __ adr(lr, __ pc());
 2375   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2376 
 2377   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
 2378   __ end_a_stub();
 2379   return offset;
 2380 }
 2381 
 2382 // REQUIRED MATCHER CODE
 2383 
 2384 //=============================================================================
 2385 
 2386 const bool Matcher::match_rule_supported(int opcode) {
 2387   if (!has_match_rule(opcode))
 2388     return false;
 2389 
 2390   bool ret_value = true;
 2391   switch (opcode) {
 2392     case Op_CacheWB:
 2393     case Op_CacheWBPreSync:
 2394     case Op_CacheWBPostSync:
 2395       if (!VM_Version::supports_data_cache_line_flush()) {
 2396         ret_value = false;
 2397       }
 2398       break;
 2399   }
 2400 
 2401   return ret_value; // Per default match rules are supported.
 2402 }
 2403 
 2404 // Identify extra cases that we might want to provide match rules for vector nodes and
 2405 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2406 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2407   if (!match_rule_supported(opcode)) {
 2408     return false;
 2409   }
 2410   int bit_size = vlen * type2aelembytes(bt) * 8;
 2411   if (UseSVE == 0 && bit_size > 128) {
 2412     return false;
 2413   }
 2414   if (UseSVE > 0) {
 2415     return op_sve_supported(opcode, vlen, bt);
 2416   } else { // NEON
 2417     // Special cases
 2418     switch (opcode) {
 2419     case Op_VectorMaskCmp:
 2420     // We don't have VectorReinterpret with bit_size less than 64 support for
 2421     // now, even for byte type. To be refined with fully VectorCast support.
 2422     case Op_VectorReinterpret:
 2423       if (vlen < 2 || bit_size < 64) {
 2424         return false;
 2425       }
 2426       break;
 2427     case Op_MulAddVS2VI:
 2428       if (bit_size < 128) {
 2429         return false;
 2430       }
 2431       break;
 2432     case Op_MulVL:
 2433       return false;
 2434     case Op_VectorLoadShuffle:
 2435     case Op_VectorRearrange:
 2436       if (vlen < 4) {
 2437         return false;
 2438       }
 2439       break;
 2440     // Some types of VectorCast are not implemented for now.
 2441     case Op_VectorCastI2X:
 2442       if (bt == T_BYTE) {
 2443         return false;
 2444       }
 2445       break;
 2446     case Op_VectorCastS2X:
 2447       if (vlen < 4 || bit_size < 64) {
 2448         return false;
 2449       }
 2450       break;
 2451     case Op_VectorCastF2X:
 2452     case Op_VectorCastD2X:
 2453       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2454         return false;
 2455       }
 2456       break;
 2457     case Op_LoadVectorGather:
 2458     case Op_StoreVectorScatter:
 2459       return false;
 2460     default:
 2461       break;
 2462     }
 2463   }
 2464   return vector_size_supported(bt, vlen);
 2465 }
 2466 
 2467 const RegMask* Matcher::predicate_reg_mask(void) {
 2468   return &_PR_REG_mask;
 2469 }
 2470 
 2471 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2472   return new TypeVectMask(elemTy, length);
 2473 }
 2474 
 2475 // Vector calling convention not yet implemented.
 2476 const bool Matcher::supports_vector_calling_convention(void) {
 2477   return false;
 2478 }
 2479 
 2480 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2481   Unimplemented();
 2482   return OptoRegPair(0, 0);
 2483 }
 2484 
 2485 // Is this branch offset short enough that a short branch can be used?
 2486 //
 2487 // NOTE: If the platform does not provide any short branch variants, then
 2488 //       this method should return false for offset 0.
 2489 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2490   // The passed offset is relative to address of the branch.
 2491 
 2492   return (-32768 <= offset && offset < 32768);
 2493 }
 2494 
 2495 // Vector width in bytes.
 2496 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2497   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2498   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2499   // Minimum 2 values in vector
 2500   if (size < 2*type2aelembytes(bt)) size = 0;
 2501   // But never < 4
 2502   if (size < 4) size = 0;
 2503   return size;
 2504 }
 2505 
 2506 // Limits on vector size (number of elements) loaded into vector.
 2507 const int Matcher::max_vector_size(const BasicType bt) {
 2508   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2509 }
 2510 
 2511 const int Matcher::min_vector_size(const BasicType bt) {
 2512   int max_size = max_vector_size(bt);
 2513   // Limit the min vector size to 8 bytes.
 2514   int size = 8 / type2aelembytes(bt);
 2515   if (bt == T_BYTE) {
 2516     // To support vector api shuffle/rearrange.
 2517     size = 4;
 2518   } else if (bt == T_BOOLEAN) {
 2519     // To support vector api load/store mask.
 2520     size = 2;
 2521   }
 2522   if (size < 2) size = 2;
 2523   return MIN2(size, max_size);
 2524 }
 2525 
 2526 // Actual max scalable vector register length.
 2527 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2528   return Matcher::max_vector_size(bt);
 2529 }
 2530 
 2531 // Vector ideal reg.
 2532 const uint Matcher::vector_ideal_reg(int len) {
 2533   if (UseSVE > 0 && 2 <= len && len <= 256) {
 2534     return Op_VecA;
 2535   }
 2536   switch(len) {
 2537     // For 16-bit/32-bit mask vector, reuse VecD.
 2538     case  2:
 2539     case  4:
 2540     case  8: return Op_VecD;
 2541     case 16: return Op_VecX;
 2542   }
 2543   ShouldNotReachHere();
 2544   return 0;
 2545 }
 2546 
 2547 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2548   ShouldNotReachHere(); // generic vector operands not supported
 2549   return NULL;
 2550 }
 2551 
 2552 bool Matcher::is_reg2reg_move(MachNode* m) {
 2553   ShouldNotReachHere();  // generic vector operands not supported
 2554   return false;
 2555 }
 2556 
 2557 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2558   ShouldNotReachHere();  // generic vector operands not supported
 2559   return false;
 2560 }
 2561 
 2562 // Return whether or not this register is ever used as an argument.
 2563 // This function is used on startup to build the trampoline stubs in
 2564 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2565 // call in the trampoline, and arguments in those registers not be
 2566 // available to the callee.
 2567 bool Matcher::can_be_java_arg(int reg)
 2568 {
 2569   return
 2570     reg ==  R0_num || reg == R0_H_num ||
 2571     reg ==  R1_num || reg == R1_H_num ||
 2572     reg ==  R2_num || reg == R2_H_num ||
 2573     reg ==  R3_num || reg == R3_H_num ||
 2574     reg ==  R4_num || reg == R4_H_num ||
 2575     reg ==  R5_num || reg == R5_H_num ||
 2576     reg ==  R6_num || reg == R6_H_num ||
 2577     reg ==  R7_num || reg == R7_H_num ||
 2578     reg ==  V0_num || reg == V0_H_num ||
 2579     reg ==  V1_num || reg == V1_H_num ||
 2580     reg ==  V2_num || reg == V2_H_num ||
 2581     reg ==  V3_num || reg == V3_H_num ||
 2582     reg ==  V4_num || reg == V4_H_num ||
 2583     reg ==  V5_num || reg == V5_H_num ||
 2584     reg ==  V6_num || reg == V6_H_num ||
 2585     reg ==  V7_num || reg == V7_H_num;
 2586 }
 2587 
 2588 bool Matcher::is_spillable_arg(int reg)
 2589 {
 2590   return can_be_java_arg(reg);
 2591 }
 2592 
 2593 uint Matcher::int_pressure_limit()
 2594 {
 2595   // JDK-8183543: When taking the number of available registers as int
 2596   // register pressure threshold, the jtreg test:
 2597   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2598   // failed due to C2 compilation failure with
 2599   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2600   //
 2601   // A derived pointer is live at CallNode and then is flagged by RA
 2602   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2603   // derived pointers and lastly fail to spill after reaching maximum
 2604   // number of iterations. Lowering the default pressure threshold to
 2605   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2606   // a high register pressure area of the code so that split_DEF can
 2607   // generate DefinitionSpillCopy for the derived pointer.
 2608   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2609   if (!PreserveFramePointer) {
 2610     // When PreserveFramePointer is off, frame pointer is allocatable,
 2611     // but different from other SOC registers, it is excluded from
 2612     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2613     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2614     // See check_pressure_at_fatproj().
 2615     default_int_pressure_threshold--;
 2616   }
 2617   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2618 }
 2619 
 2620 uint Matcher::float_pressure_limit()
 2621 {
 2622   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2623   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2624 }
 2625 
 2626 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2627   return false;
 2628 }
 2629 
 2630 RegMask Matcher::divI_proj_mask() {
 2631   ShouldNotReachHere();
 2632   return RegMask();
 2633 }
 2634 
 2635 // Register for MODI projection of divmodI.
 2636 RegMask Matcher::modI_proj_mask() {
 2637   ShouldNotReachHere();
 2638   return RegMask();
 2639 }
 2640 
 2641 // Register for DIVL projection of divmodL.
 2642 RegMask Matcher::divL_proj_mask() {
 2643   ShouldNotReachHere();
 2644   return RegMask();
 2645 }
 2646 
 2647 // Register for MODL projection of divmodL.
 2648 RegMask Matcher::modL_proj_mask() {
 2649   ShouldNotReachHere();
 2650   return RegMask();
 2651 }
 2652 
 2653 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2654   return FP_REG_mask();
 2655 }
 2656 
 2657 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2658   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2659     Node* u = addp->fast_out(i);
 2660     if (u->is_LoadStore()) {
 2661       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2662       // instructions) only take register indirect as an operand, so
 2663       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2664       // must fail.
 2665       return false;
 2666     }
 2667     if (u->is_Mem()) {
 2668       int opsize = u->as_Mem()->memory_size();
 2669       assert(opsize > 0, "unexpected memory operand size");
 2670       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2671         return false;
 2672       }
 2673     }
 2674   }
 2675   return true;
 2676 }
 2677 
 2678 // Should the matcher clone input 'm' of node 'n'?
 2679 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2680   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2681     mstack.push(m, Visit);           // m = ShiftCntV
 2682     return true;
 2683   }
 2684   return false;
 2685 }
 2686 
 2687 // Should the Matcher clone shifts on addressing modes, expecting them
 2688 // to be subsumed into complex addressing expressions or compute them
 2689 // into registers?
 2690 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2691   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2692     return true;
 2693   }
 2694 
 2695   Node *off = m->in(AddPNode::Offset);
 2696   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2697       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2698       // Are there other uses besides address expressions?
 2699       !is_visited(off)) {
 2700     address_visited.set(off->_idx); // Flag as address_visited
 2701     mstack.push(off->in(2), Visit);
 2702     Node *conv = off->in(1);
 2703     if (conv->Opcode() == Op_ConvI2L &&
 2704         // Are there other uses besides address expressions?
 2705         !is_visited(conv)) {
 2706       address_visited.set(conv->_idx); // Flag as address_visited
 2707       mstack.push(conv->in(1), Pre_Visit);
 2708     } else {
 2709       mstack.push(conv, Pre_Visit);
 2710     }
 2711     address_visited.test_set(m->_idx); // Flag as address_visited
 2712     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2713     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2714     return true;
 2715   } else if (off->Opcode() == Op_ConvI2L &&
 2716              // Are there other uses besides address expressions?
 2717              !is_visited(off)) {
 2718     address_visited.test_set(m->_idx); // Flag as address_visited
 2719     address_visited.set(off->_idx); // Flag as address_visited
 2720     mstack.push(off->in(1), Pre_Visit);
 2721     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2722     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2723     return true;
 2724   }
 2725   return false;
 2726 }
 2727 
 2728 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2729   C2_MacroAssembler _masm(&cbuf);                                       \
 2730   {                                                                     \
 2731     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2732     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2733     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2734     __ INSN(REG, as_Register(BASE));                                    \
 2735   }
 2736 
 2737 
 2738 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2739   {
 2740     Address::extend scale;
 2741 
 2742     // Hooboy, this is fugly.  We need a way to communicate to the
 2743     // encoder that the index needs to be sign extended, so we have to
 2744     // enumerate all the cases.
 2745     switch (opcode) {
 2746     case INDINDEXSCALEDI2L:
 2747     case INDINDEXSCALEDI2LN:
 2748     case INDINDEXI2L:
 2749     case INDINDEXI2LN:
 2750       scale = Address::sxtw(size);
 2751       break;
 2752     default:
 2753       scale = Address::lsl(size);
 2754     }
 2755 
 2756     if (index == -1) {
 2757       return Address(base, disp);
 2758     } else {
 2759       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2760       return Address(base, as_Register(index), scale);
 2761     }
 2762   }
 2763 
 2764 
 2765 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2766 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2767 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2768 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2769                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2770 
 2771   // Used for all non-volatile memory accesses.  The use of
 2772   // $mem->opcode() to discover whether this pattern uses sign-extended
 2773   // offsets is something of a kludge.
 2774   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2775                         Register reg, int opcode,
 2776                         Register base, int index, int scale, int disp,
 2777                         int size_in_memory)
 2778   {
 2779     Address addr = mem2address(opcode, base, index, scale, disp);
 2780     if (addr.getMode() == Address::base_plus_offset) {
 2781       /* If we get an out-of-range offset it is a bug in the compiler,
 2782          so we assert here. */
 2783       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2784              "c2 compiler bug");
 2785       /* Fix up any out-of-range offsets. */
 2786       assert_different_registers(rscratch1, base);
 2787       assert_different_registers(rscratch1, reg);
 2788       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2789     }
 2790     (masm.*insn)(reg, addr);
 2791   }
 2792 
 2793   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2794                         FloatRegister reg, int opcode,
 2795                         Register base, int index, int size, int disp,
 2796                         int size_in_memory)
 2797   {
 2798     Address::extend scale;
 2799 
 2800     switch (opcode) {
 2801     case INDINDEXSCALEDI2L:
 2802     case INDINDEXSCALEDI2LN:
 2803       scale = Address::sxtw(size);
 2804       break;
 2805     default:
 2806       scale = Address::lsl(size);
 2807     }
 2808 
 2809     if (index == -1) {
 2810       /* If we get an out-of-range offset it is a bug in the compiler,
 2811          so we assert here. */
 2812       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2813       /* Fix up any out-of-range offsets. */
 2814       assert_different_registers(rscratch1, base);
 2815       Address addr = Address(base, disp);
 2816       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2817       (masm.*insn)(reg, addr);
 2818     } else {
 2819       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2820       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2821     }
 2822   }
 2823 
 2824   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2825                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2826                         int opcode, Register base, int index, int size, int disp)
 2827   {
 2828     if (index == -1) {
 2829       (masm.*insn)(reg, T, Address(base, disp));
 2830     } else {
 2831       assert(disp == 0, "unsupported address mode");
 2832       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2833     }
 2834   }
 2835 
 2836 %}
 2837 
 2838 
 2839 
 2840 //----------ENCODING BLOCK-----------------------------------------------------
 2841 // This block specifies the encoding classes used by the compiler to
 2842 // output byte streams.  Encoding classes are parameterized macros
 2843 // used by Machine Instruction Nodes in order to generate the bit
 2844 // encoding of the instruction.  Operands specify their base encoding
 2845 // interface with the interface keyword.  There are currently
 2846 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2847 // COND_INTER.  REG_INTER causes an operand to generate a function
 2848 // which returns its register number when queried.  CONST_INTER causes
 2849 // an operand to generate a function which returns the value of the
 2850 // constant when queried.  MEMORY_INTER causes an operand to generate
 2851 // four functions which return the Base Register, the Index Register,
 2852 // the Scale Value, and the Offset Value of the operand when queried.
 2853 // COND_INTER causes an operand to generate six functions which return
 2854 // the encoding code (ie - encoding bits for the instruction)
 2855 // associated with each basic boolean condition for a conditional
 2856 // instruction.
 2857 //
 2858 // Instructions specify two basic values for encoding.  Again, a
 2859 // function is available to check if the constant displacement is an
 2860 // oop. They use the ins_encode keyword to specify their encoding
 2861 // classes (which must be a sequence of enc_class names, and their
 2862 // parameters, specified in the encoding block), and they use the
 2863 // opcode keyword to specify, in order, their primary, secondary, and
 2864 // tertiary opcode.  Only the opcode sections which a particular
 2865 // instruction needs for encoding need to be specified.
 2866 encode %{
 2867   // Build emit functions for each basic byte or larger field in the
 2868   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2869   // from C++ code in the enc_class source block.  Emit functions will
 2870   // live in the main source block for now.  In future, we can
 2871   // generalize this by adding a syntax that specifies the sizes of
 2872   // fields in an order, so that the adlc can build the emit functions
 2873   // automagically
 2874 
 2875   // catch all for unimplemented encodings
 2876   enc_class enc_unimplemented %{
 2877     C2_MacroAssembler _masm(&cbuf);
 2878     __ unimplemented("C2 catch all");
 2879   %}
 2880 
 2881   // BEGIN Non-volatile memory access
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2926     Register dst_reg = as_Register($dst$$reg);
 2927     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2934     Register dst_reg = as_Register($dst$$reg);
 2935     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2942     Register dst_reg = as_Register($dst$$reg);
 2943     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2950     Register dst_reg = as_Register($dst$$reg);
 2951     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2958     Register dst_reg = as_Register($dst$$reg);
 2959     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2966     Register dst_reg = as_Register($dst$$reg);
 2967     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2968                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2969   %}
 2970 
 2971   // This encoding class is generated automatically from ad_encode.m4.
 2972   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2973   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2974     Register dst_reg = as_Register($dst$$reg);
 2975     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2976                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2977   %}
 2978 
 2979   // This encoding class is generated automatically from ad_encode.m4.
 2980   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2981   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2982     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2983     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2984                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2985   %}
 2986 
 2987   // This encoding class is generated automatically from ad_encode.m4.
 2988   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2989   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2990     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2991     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2998     Register src_reg = as_Register($src$$reg);
 2999     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_strb0(memory1 mem) %{
 3006     C2_MacroAssembler _masm(&cbuf);
 3007     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3008                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3009   %}
 3010 
 3011   // This encoding class is generated automatically from ad_encode.m4.
 3012   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3013   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3014     Register src_reg = as_Register($src$$reg);
 3015     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_strh0(memory2 mem) %{
 3022     C2_MacroAssembler _masm(&cbuf);
 3023     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3024                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3025   %}
 3026 
 3027   // This encoding class is generated automatically from ad_encode.m4.
 3028   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3029   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3030     Register src_reg = as_Register($src$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 3032                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3033   %}
 3034 
 3035   // This encoding class is generated automatically from ad_encode.m4.
 3036   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3037   enc_class aarch64_enc_strw0(memory4 mem) %{
 3038     C2_MacroAssembler _masm(&cbuf);
 3039     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3041   %}
 3042 
 3043   // This encoding class is generated automatically from ad_encode.m4.
 3044   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3045   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3046     Register src_reg = as_Register($src$$reg);
 3047     // we sometimes get asked to store the stack pointer into the
 3048     // current thread -- we cannot do that directly on AArch64
 3049     if (src_reg == r31_sp) {
 3050       C2_MacroAssembler _masm(&cbuf);
 3051       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3052       __ mov(rscratch2, sp);
 3053       src_reg = rscratch2;
 3054     }
 3055     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3056                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3057   %}
 3058 
 3059   // This encoding class is generated automatically from ad_encode.m4.
 3060   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3061   enc_class aarch64_enc_str0(memory8 mem) %{
 3062     C2_MacroAssembler _masm(&cbuf);
 3063     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3064                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3065   %}
 3066 
 3067   // This encoding class is generated automatically from ad_encode.m4.
 3068   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3069   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3070     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3071     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3072                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3073   %}
 3074 
 3075   // This encoding class is generated automatically from ad_encode.m4.
 3076   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3077   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3078     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3079     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3080                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3081   %}
 3082 
 3083   // This encoding class is generated automatically from ad_encode.m4.
 3084   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3085   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3086       C2_MacroAssembler _masm(&cbuf);
 3087       __ membar(Assembler::StoreStore);
 3088       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3089                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3090   %}
 3091 
 3092   // END Non-volatile memory access
 3093 
 3094   // Vector loads and stores
 3095   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3096     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3097     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3098        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3102     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3103     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3104        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3105   %}
 3106 
 3107   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3108     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3109     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3110        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3114     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3115     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3116        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3117   %}
 3118 
 3119   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3120     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3121     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3122        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3123   %}
 3124 
 3125   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3126     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3127     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3128        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3129   %}
 3130 
 3131   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3132     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3133     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3134        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3138     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3139     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3140        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3141   %}
 3142 
 3143   // volatile loads and stores
 3144 
 3145   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3146     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3147                  rscratch1, stlrb);
 3148   %}
 3149 
 3150   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3151     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3152                  rscratch1, stlrh);
 3153   %}
 3154 
 3155   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3156     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3157                  rscratch1, stlrw);
 3158   %}
 3159 
 3160 
 3161   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3162     Register dst_reg = as_Register($dst$$reg);
 3163     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3164              rscratch1, ldarb);
 3165     __ sxtbw(dst_reg, dst_reg);
 3166   %}
 3167 
 3168   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3169     Register dst_reg = as_Register($dst$$reg);
 3170     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171              rscratch1, ldarb);
 3172     __ sxtb(dst_reg, dst_reg);
 3173   %}
 3174 
 3175   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3176     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3177              rscratch1, ldarb);
 3178   %}
 3179 
 3180   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3181     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3182              rscratch1, ldarb);
 3183   %}
 3184 
 3185   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3186     Register dst_reg = as_Register($dst$$reg);
 3187     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3188              rscratch1, ldarh);
 3189     __ sxthw(dst_reg, dst_reg);
 3190   %}
 3191 
 3192   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3193     Register dst_reg = as_Register($dst$$reg);
 3194     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3195              rscratch1, ldarh);
 3196     __ sxth(dst_reg, dst_reg);
 3197   %}
 3198 
 3199   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3200     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3201              rscratch1, ldarh);
 3202   %}
 3203 
 3204   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3205     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3206              rscratch1, ldarh);
 3207   %}
 3208 
 3209   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3210     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3211              rscratch1, ldarw);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3215     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3216              rscratch1, ldarw);
 3217   %}
 3218 
 3219   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3220     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3221              rscratch1, ldar);
 3222   %}
 3223 
 3224   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3225     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3226              rscratch1, ldarw);
 3227     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3231     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3232              rscratch1, ldar);
 3233     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3234   %}
 3235 
 3236   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3237     Register src_reg = as_Register($src$$reg);
 3238     // we sometimes get asked to store the stack pointer into the
 3239     // current thread -- we cannot do that directly on AArch64
 3240     if (src_reg == r31_sp) {
 3241       C2_MacroAssembler _masm(&cbuf);
 3242       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3243       __ mov(rscratch2, sp);
 3244       src_reg = rscratch2;
 3245     }
 3246     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3247                  rscratch1, stlr);
 3248   %}
 3249 
 3250   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3251     {
 3252       C2_MacroAssembler _masm(&cbuf);
 3253       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3254       __ fmovs(rscratch2, src_reg);
 3255     }
 3256     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3257                  rscratch1, stlrw);
 3258   %}
 3259 
 3260   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3261     {
 3262       C2_MacroAssembler _masm(&cbuf);
 3263       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3264       __ fmovd(rscratch2, src_reg);
 3265     }
 3266     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3267                  rscratch1, stlr);
 3268   %}
 3269 
 3270   // synchronized read/update encodings
 3271 
 3272   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3273     C2_MacroAssembler _masm(&cbuf);
 3274     Register dst_reg = as_Register($dst$$reg);
 3275     Register base = as_Register($mem$$base);
 3276     int index = $mem$$index;
 3277     int scale = $mem$$scale;
 3278     int disp = $mem$$disp;
 3279     if (index == -1) {
 3280        if (disp != 0) {
 3281         __ lea(rscratch1, Address(base, disp));
 3282         __ ldaxr(dst_reg, rscratch1);
 3283       } else {
 3284         // TODO
 3285         // should we ever get anything other than this case?
 3286         __ ldaxr(dst_reg, base);
 3287       }
 3288     } else {
 3289       Register index_reg = as_Register(index);
 3290       if (disp == 0) {
 3291         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3292         __ ldaxr(dst_reg, rscratch1);
 3293       } else {
 3294         __ lea(rscratch1, Address(base, disp));
 3295         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3296         __ ldaxr(dst_reg, rscratch1);
 3297       }
 3298     }
 3299   %}
 3300 
 3301   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3302     C2_MacroAssembler _masm(&cbuf);
 3303     Register src_reg = as_Register($src$$reg);
 3304     Register base = as_Register($mem$$base);
 3305     int index = $mem$$index;
 3306     int scale = $mem$$scale;
 3307     int disp = $mem$$disp;
 3308     if (index == -1) {
 3309        if (disp != 0) {
 3310         __ lea(rscratch2, Address(base, disp));
 3311         __ stlxr(rscratch1, src_reg, rscratch2);
 3312       } else {
 3313         // TODO
 3314         // should we ever get anything other than this case?
 3315         __ stlxr(rscratch1, src_reg, base);
 3316       }
 3317     } else {
 3318       Register index_reg = as_Register(index);
 3319       if (disp == 0) {
 3320         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3321         __ stlxr(rscratch1, src_reg, rscratch2);
 3322       } else {
 3323         __ lea(rscratch2, Address(base, disp));
 3324         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3325         __ stlxr(rscratch1, src_reg, rscratch2);
 3326       }
 3327     }
 3328     __ cmpw(rscratch1, zr);
 3329   %}
 3330 
 3331   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3332     C2_MacroAssembler _masm(&cbuf);
 3333     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3334     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3335                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3336                /*weak*/ false, noreg);
 3337   %}
 3338 
 3339   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3340     C2_MacroAssembler _masm(&cbuf);
 3341     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3342     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3343                Assembler::word, /*acquire*/ false, /*release*/ true,
 3344                /*weak*/ false, noreg);
 3345   %}
 3346 
 3347   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3348     C2_MacroAssembler _masm(&cbuf);
 3349     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3350     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3351                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3352                /*weak*/ false, noreg);
 3353   %}
 3354 
 3355   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3356     C2_MacroAssembler _masm(&cbuf);
 3357     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3358     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3359                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3360                /*weak*/ false, noreg);
 3361   %}
 3362 
 3363 
 3364   // The only difference between aarch64_enc_cmpxchg and
 3365   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3366   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3367   // lock.
 3368   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3369     C2_MacroAssembler _masm(&cbuf);
 3370     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3371     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3372                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3373                /*weak*/ false, noreg);
 3374   %}
 3375 
 3376   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3377     C2_MacroAssembler _masm(&cbuf);
 3378     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3379     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3380                Assembler::word, /*acquire*/ true, /*release*/ true,
 3381                /*weak*/ false, noreg);
 3382   %}
 3383 
 3384   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3385     C2_MacroAssembler _masm(&cbuf);
 3386     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3387     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3388                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3389                /*weak*/ false, noreg);
 3390   %}
 3391 
 3392   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3393     C2_MacroAssembler _masm(&cbuf);
 3394     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3395     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3396                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3397                /*weak*/ false, noreg);
 3398   %}
 3399 
 3400   // auxiliary used for CompareAndSwapX to set result register
 3401   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3402     C2_MacroAssembler _masm(&cbuf);
 3403     Register res_reg = as_Register($res$$reg);
 3404     __ cset(res_reg, Assembler::EQ);
 3405   %}
 3406 
 3407   // prefetch encodings
 3408 
 3409   enc_class aarch64_enc_prefetchw(memory mem) %{
 3410     C2_MacroAssembler _masm(&cbuf);
 3411     Register base = as_Register($mem$$base);
 3412     int index = $mem$$index;
 3413     int scale = $mem$$scale;
 3414     int disp = $mem$$disp;
 3415     if (index == -1) {
 3416       __ prfm(Address(base, disp), PSTL1KEEP);
 3417     } else {
 3418       Register index_reg = as_Register(index);
 3419       if (disp == 0) {
 3420         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3421       } else {
 3422         __ lea(rscratch1, Address(base, disp));
 3423 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3424       }
 3425     }
 3426   %}
 3427 
 3428   /// mov envcodings
 3429 
 3430   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3431     C2_MacroAssembler _masm(&cbuf);
 3432     uint32_t con = (uint32_t)$src$$constant;
 3433     Register dst_reg = as_Register($dst$$reg);
 3434     if (con == 0) {
 3435       __ movw(dst_reg, zr);
 3436     } else {
 3437       __ movw(dst_reg, con);
 3438     }
 3439   %}
 3440 
 3441   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3442     C2_MacroAssembler _masm(&cbuf);
 3443     Register dst_reg = as_Register($dst$$reg);
 3444     uint64_t con = (uint64_t)$src$$constant;
 3445     if (con == 0) {
 3446       __ mov(dst_reg, zr);
 3447     } else {
 3448       __ mov(dst_reg, con);
 3449     }
 3450   %}
 3451 
 3452   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3453     C2_MacroAssembler _masm(&cbuf);
 3454     Register dst_reg = as_Register($dst$$reg);
 3455     address con = (address)$src$$constant;
 3456     if (con == NULL || con == (address)1) {
 3457       ShouldNotReachHere();
 3458     } else {
 3459       relocInfo::relocType rtype = $src->constant_reloc();
 3460       if (rtype == relocInfo::oop_type) {
 3461         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3462       } else if (rtype == relocInfo::metadata_type) {
 3463         __ mov_metadata(dst_reg, (Metadata*)con);
 3464       } else {
 3465         assert(rtype == relocInfo::none, "unexpected reloc type");
 3466         if (con < (address)(uintptr_t)os::vm_page_size()) {
 3467           __ mov(dst_reg, con);
 3468         } else {
 3469           uint64_t offset;
 3470           __ adrp(dst_reg, con, offset);
 3471           __ add(dst_reg, dst_reg, offset);
 3472         }
 3473       }
 3474     }
 3475   %}
 3476 
 3477   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3478     C2_MacroAssembler _masm(&cbuf);
 3479     Register dst_reg = as_Register($dst$$reg);
 3480     __ mov(dst_reg, zr);
 3481   %}
 3482 
 3483   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3484     C2_MacroAssembler _masm(&cbuf);
 3485     Register dst_reg = as_Register($dst$$reg);
 3486     __ mov(dst_reg, (uint64_t)1);
 3487   %}
 3488 
 3489   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3490     C2_MacroAssembler _masm(&cbuf);
 3491     __ load_byte_map_base($dst$$Register);
 3492   %}
 3493 
 3494   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3495     C2_MacroAssembler _masm(&cbuf);
 3496     Register dst_reg = as_Register($dst$$reg);
 3497     address con = (address)$src$$constant;
 3498     if (con == NULL) {
 3499       ShouldNotReachHere();
 3500     } else {
 3501       relocInfo::relocType rtype = $src->constant_reloc();
 3502       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3503       __ set_narrow_oop(dst_reg, (jobject)con);
 3504     }
 3505   %}
 3506 
 3507   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3508     C2_MacroAssembler _masm(&cbuf);
 3509     Register dst_reg = as_Register($dst$$reg);
 3510     __ mov(dst_reg, zr);
 3511   %}
 3512 
 3513   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3514     C2_MacroAssembler _masm(&cbuf);
 3515     Register dst_reg = as_Register($dst$$reg);
 3516     address con = (address)$src$$constant;
 3517     if (con == NULL) {
 3518       ShouldNotReachHere();
 3519     } else {
 3520       relocInfo::relocType rtype = $src->constant_reloc();
 3521       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3522       __ set_narrow_klass(dst_reg, (Klass *)con);
 3523     }
 3524   %}
 3525 
 3526   // arithmetic encodings
 3527 
 3528   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3529     C2_MacroAssembler _masm(&cbuf);
 3530     Register dst_reg = as_Register($dst$$reg);
 3531     Register src_reg = as_Register($src1$$reg);
 3532     int32_t con = (int32_t)$src2$$constant;
 3533     // add has primary == 0, subtract has primary == 1
 3534     if ($primary) { con = -con; }
 3535     if (con < 0) {
 3536       __ subw(dst_reg, src_reg, -con);
 3537     } else {
 3538       __ addw(dst_reg, src_reg, con);
 3539     }
 3540   %}
 3541 
 3542   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3543     C2_MacroAssembler _masm(&cbuf);
 3544     Register dst_reg = as_Register($dst$$reg);
 3545     Register src_reg = as_Register($src1$$reg);
 3546     int32_t con = (int32_t)$src2$$constant;
 3547     // add has primary == 0, subtract has primary == 1
 3548     if ($primary) { con = -con; }
 3549     if (con < 0) {
 3550       __ sub(dst_reg, src_reg, -con);
 3551     } else {
 3552       __ add(dst_reg, src_reg, con);
 3553     }
 3554   %}
 3555 
 3556   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558    Register dst_reg = as_Register($dst$$reg);
 3559    Register src1_reg = as_Register($src1$$reg);
 3560    Register src2_reg = as_Register($src2$$reg);
 3561     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3565     C2_MacroAssembler _masm(&cbuf);
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3573     C2_MacroAssembler _masm(&cbuf);
 3574    Register dst_reg = as_Register($dst$$reg);
 3575    Register src1_reg = as_Register($src1$$reg);
 3576    Register src2_reg = as_Register($src2$$reg);
 3577     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3578   %}
 3579 
 3580   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3581     C2_MacroAssembler _masm(&cbuf);
 3582    Register dst_reg = as_Register($dst$$reg);
 3583    Register src1_reg = as_Register($src1$$reg);
 3584    Register src2_reg = as_Register($src2$$reg);
 3585     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3586   %}
 3587 
 3588   // compare instruction encodings
 3589 
 3590   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3591     C2_MacroAssembler _masm(&cbuf);
 3592     Register reg1 = as_Register($src1$$reg);
 3593     Register reg2 = as_Register($src2$$reg);
 3594     __ cmpw(reg1, reg2);
 3595   %}
 3596 
 3597   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3598     C2_MacroAssembler _masm(&cbuf);
 3599     Register reg = as_Register($src1$$reg);
 3600     int32_t val = $src2$$constant;
 3601     if (val >= 0) {
 3602       __ subsw(zr, reg, val);
 3603     } else {
 3604       __ addsw(zr, reg, -val);
 3605     }
 3606   %}
 3607 
 3608   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3609     C2_MacroAssembler _masm(&cbuf);
 3610     Register reg1 = as_Register($src1$$reg);
 3611     uint32_t val = (uint32_t)$src2$$constant;
 3612     __ movw(rscratch1, val);
 3613     __ cmpw(reg1, rscratch1);
 3614   %}
 3615 
 3616   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3617     C2_MacroAssembler _masm(&cbuf);
 3618     Register reg1 = as_Register($src1$$reg);
 3619     Register reg2 = as_Register($src2$$reg);
 3620     __ cmp(reg1, reg2);
 3621   %}
 3622 
 3623   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3624     C2_MacroAssembler _masm(&cbuf);
 3625     Register reg = as_Register($src1$$reg);
 3626     int64_t val = $src2$$constant;
 3627     if (val >= 0) {
 3628       __ subs(zr, reg, val);
 3629     } else if (val != -val) {
 3630       __ adds(zr, reg, -val);
 3631     } else {
 3632     // aargh, Long.MIN_VALUE is a special case
 3633       __ orr(rscratch1, zr, (uint64_t)val);
 3634       __ subs(zr, reg, rscratch1);
 3635     }
 3636   %}
 3637 
 3638   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3639     C2_MacroAssembler _masm(&cbuf);
 3640     Register reg1 = as_Register($src1$$reg);
 3641     uint64_t val = (uint64_t)$src2$$constant;
 3642     __ mov(rscratch1, val);
 3643     __ cmp(reg1, rscratch1);
 3644   %}
 3645 
 3646   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3647     C2_MacroAssembler _masm(&cbuf);
 3648     Register reg1 = as_Register($src1$$reg);
 3649     Register reg2 = as_Register($src2$$reg);
 3650     __ cmp(reg1, reg2);
 3651   %}
 3652 
 3653   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3654     C2_MacroAssembler _masm(&cbuf);
 3655     Register reg1 = as_Register($src1$$reg);
 3656     Register reg2 = as_Register($src2$$reg);
 3657     __ cmpw(reg1, reg2);
 3658   %}
 3659 
 3660   enc_class aarch64_enc_testp(iRegP src) %{
 3661     C2_MacroAssembler _masm(&cbuf);
 3662     Register reg = as_Register($src$$reg);
 3663     __ cmp(reg, zr);
 3664   %}
 3665 
 3666   enc_class aarch64_enc_testn(iRegN src) %{
 3667     C2_MacroAssembler _masm(&cbuf);
 3668     Register reg = as_Register($src$$reg);
 3669     __ cmpw(reg, zr);
 3670   %}
 3671 
 3672   enc_class aarch64_enc_b(label lbl) %{
 3673     C2_MacroAssembler _masm(&cbuf);
 3674     Label *L = $lbl$$label;
 3675     __ b(*L);
 3676   %}
 3677 
 3678   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3679     C2_MacroAssembler _masm(&cbuf);
 3680     Label *L = $lbl$$label;
 3681     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3682   %}
 3683 
 3684   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3685     C2_MacroAssembler _masm(&cbuf);
 3686     Label *L = $lbl$$label;
 3687     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3688   %}
 3689 
 3690   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3691   %{
 3692      Register sub_reg = as_Register($sub$$reg);
 3693      Register super_reg = as_Register($super$$reg);
 3694      Register temp_reg = as_Register($temp$$reg);
 3695      Register result_reg = as_Register($result$$reg);
 3696 
 3697      Label miss;
 3698      C2_MacroAssembler _masm(&cbuf);
 3699      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3700                                      NULL, &miss,
 3701                                      /*set_cond_codes:*/ true);
 3702      if ($primary) {
 3703        __ mov(result_reg, zr);
 3704      }
 3705      __ bind(miss);
 3706   %}
 3707 
 3708   enc_class aarch64_enc_java_static_call(method meth) %{
 3709     C2_MacroAssembler _masm(&cbuf);
 3710 
 3711     address addr = (address)$meth$$method;
 3712     address call;
 3713     if (!_method) {
 3714       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3715       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3716       if (call == NULL) {
 3717         ciEnv::current()->record_failure("CodeCache is full");
 3718         return;
 3719       }
 3720     } else {
 3721       int method_index = resolved_method_index(cbuf);
 3722       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3723                                                   : static_call_Relocation::spec(method_index);
 3724       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3725       if (call == NULL) {
 3726         ciEnv::current()->record_failure("CodeCache is full");
 3727         return;
 3728       }
 3729       // Emit stub for static call
 3730       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3731       if (stub == NULL) {
 3732         ciEnv::current()->record_failure("CodeCache is full");
 3733         return;
 3734       }
 3735     }
 3736 
 3737     // Only non uncommon_trap calls need to reinitialize ptrue.
 3738     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3739       __ reinitialize_ptrue();
 3740     }
 3741   %}
 3742 
 3743   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3744     C2_MacroAssembler _masm(&cbuf);
 3745     int method_index = resolved_method_index(cbuf);
 3746     address call = __ ic_call((address)$meth$$method, method_index);
 3747     if (call == NULL) {
 3748       ciEnv::current()->record_failure("CodeCache is full");
 3749       return;
 3750     } else if (Compile::current()->max_vector_size() > 0) {
 3751       __ reinitialize_ptrue();
 3752     }
 3753   %}
 3754 
 3755   enc_class aarch64_enc_call_epilog() %{
 3756     C2_MacroAssembler _masm(&cbuf);
 3757     if (VerifyStackAtCalls) {
 3758       // Check that stack depth is unchanged: find majik cookie on stack
 3759       __ call_Unimplemented();
 3760     }
 3761     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3762       // An inline type is returned as fields in multiple registers.
 3763       // R0 either contains an oop if the inline type is buffered or a pointer
 3764       // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3765       // if the lowest bit is set to allow C2 to use the oop after null checking.
 3766       // r0 &= (r0 & 1) - 1
 3767       C2_MacroAssembler _masm(&cbuf);
 3768       __ andr(rscratch1, r0, 0x1);
 3769       __ sub(rscratch1, rscratch1, 0x1);
 3770       __ andr(r0, r0, rscratch1);
 3771     }
 3772   %}
 3773 
 3774   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3775     C2_MacroAssembler _masm(&cbuf);
 3776 
 3777     // some calls to generated routines (arraycopy code) are scheduled
 3778     // by C2 as runtime calls. if so we can call them using a br (they
 3779     // will be in a reachable segment) otherwise we have to use a blr
 3780     // which loads the absolute address into a register.
 3781     address entry = (address)$meth$$method;
 3782     CodeBlob *cb = CodeCache::find_blob(entry);
 3783     if (cb) {
 3784       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3785       if (call == NULL) {
 3786         ciEnv::current()->record_failure("CodeCache is full");
 3787         return;
 3788       }
 3789     } else {
 3790       Label retaddr;
 3791       __ adr(rscratch2, retaddr);
 3792       __ lea(rscratch1, RuntimeAddress(entry));
 3793       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3794       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3795       __ blr(rscratch1);
 3796       __ bind(retaddr);
 3797       __ add(sp, sp, 2 * wordSize);
 3798     }
 3799     if (Compile::current()->max_vector_size() > 0) {
 3800       __ reinitialize_ptrue();
 3801     }
 3802   %}
 3803 
 3804   enc_class aarch64_enc_rethrow() %{
 3805     C2_MacroAssembler _masm(&cbuf);
 3806     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3807   %}
 3808 
 3809   enc_class aarch64_enc_ret() %{
 3810     C2_MacroAssembler _masm(&cbuf);
 3811 #ifdef ASSERT
 3812     if (Compile::current()->max_vector_size() > 0) {
 3813       __ verify_ptrue();
 3814     }
 3815 #endif
 3816     __ ret(lr);
 3817   %}
 3818 
 3819   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3820     C2_MacroAssembler _masm(&cbuf);
 3821     Register target_reg = as_Register($jump_target$$reg);
 3822     __ br(target_reg);
 3823   %}
 3824 
 3825   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3826     C2_MacroAssembler _masm(&cbuf);
 3827     Register target_reg = as_Register($jump_target$$reg);
 3828     // exception oop should be in r0
 3829     // ret addr has been popped into lr
 3830     // callee expects it in r3
 3831     __ mov(r3, lr);
 3832     __ br(target_reg);
 3833   %}
 3834 
 3835   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3836     C2_MacroAssembler _masm(&cbuf);
 3837     Register oop = as_Register($object$$reg);
 3838     Register box = as_Register($box$$reg);
 3839     Register disp_hdr = as_Register($tmp$$reg);
 3840     Register tmp = as_Register($tmp2$$reg);
 3841     Label cont;
 3842     Label object_has_monitor;
 3843     Label cas_failed;
 3844 
 3845     assert_different_registers(oop, box, tmp, disp_hdr);
 3846 
 3847     // Load markWord from object into displaced_header.
 3848     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3849 
 3850     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3851       __ load_klass(tmp, oop);
 3852       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3853       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3854       __ br(Assembler::NE, cont);
 3855     }
 3856 
 3857     // Check for existing monitor
 3858     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3859 
 3860     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3861     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3862 
 3863     if (EnableValhalla) {
 3864       // Mask inline_type bit such that we go to the slow path if object is an inline type
 3865       __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
 3866     }
 3867 
 3868     // Initialize the box. (Must happen before we update the object mark!)
 3869     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3870 
 3871     // Compare object markWord with an unlocked value (tmp) and if
 3872     // equal exchange the stack address of our box with object markWord.
 3873     // On failure disp_hdr contains the possibly locked markWord.
 3874     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3875                /*release*/ true, /*weak*/ false, disp_hdr);
 3876     __ br(Assembler::EQ, cont);
 3877 
 3878     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3879 
 3880     // If the compare-and-exchange succeeded, then we found an unlocked
 3881     // object, will have now locked it will continue at label cont
 3882 
 3883     __ bind(cas_failed);
 3884     // We did not see an unlocked object so try the fast recursive case.
 3885 
 3886     // Check if the owner is self by comparing the value in the
 3887     // markWord of object (disp_hdr) with the stack pointer.
 3888     __ mov(rscratch1, sp);
 3889     __ sub(disp_hdr, disp_hdr, rscratch1);
 3890     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3891     // If condition is true we are cont and hence we can store 0 as the
 3892     // displaced header in the box, which indicates that it is a recursive lock.
 3893     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3894     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3895 
 3896     __ b(cont);
 3897 
 3898     // Handle existing monitor.
 3899     __ bind(object_has_monitor);
 3900 
 3901     // The object's monitor m is unlocked iff m->owner == NULL,
 3902     // otherwise m->owner may contain a thread or a stack address.
 3903     //
 3904     // Try to CAS m->owner from NULL to current thread.
 3905     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3906     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3907                /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
 3908 
 3909     // Store a non-null value into the box to avoid looking like a re-entrant
 3910     // lock. The fast-path monitor unlock code checks for
 3911     // markWord::monitor_value so use markWord::unused_mark which has the
 3912     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3913     __ mov(tmp, (address)markWord::unused_mark().value());
 3914     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3915 
 3916     __ bind(cont);
 3917     // flag == EQ indicates success
 3918     // flag == NE indicates failure
 3919   %}
 3920 
 3921   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3922     C2_MacroAssembler _masm(&cbuf);
 3923     Register oop = as_Register($object$$reg);
 3924     Register box = as_Register($box$$reg);
 3925     Register disp_hdr = as_Register($tmp$$reg);
 3926     Register tmp = as_Register($tmp2$$reg);
 3927     Label cont;
 3928     Label object_has_monitor;
 3929 
 3930     assert_different_registers(oop, box, tmp, disp_hdr);
 3931 
 3932     // Find the lock address and load the displaced header from the stack.
 3933     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3934 
 3935     // If the displaced header is 0, we have a recursive unlock.
 3936     __ cmp(disp_hdr, zr);
 3937     __ br(Assembler::EQ, cont);
 3938 
 3939     // Handle existing monitor.
 3940     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3941     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3942 
 3943     // Check if it is still a light weight lock, this is is true if we
 3944     // see the stack address of the basicLock in the markWord of the
 3945     // object.
 3946 
 3947     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3948                /*release*/ true, /*weak*/ false, tmp);
 3949     __ b(cont);
 3950 
 3951     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3952 
 3953     // Handle existing monitor.
 3954     __ bind(object_has_monitor);
 3955     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3956     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3957     __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3958     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3959     __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
 3960     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
 3961     __ cmp(rscratch1, zr); // Sets flags for result
 3962     __ br(Assembler::NE, cont);
 3963 
 3964     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3965     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3966     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3967     __ cmp(rscratch1, zr); // Sets flags for result
 3968     __ cbnz(rscratch1, cont);
 3969     // need a release store here
 3970     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3971     __ stlr(zr, tmp); // set unowned
 3972 
 3973     __ bind(cont);
 3974     // flag == EQ indicates success
 3975     // flag == NE indicates failure
 3976   %}
 3977 
 3978 %}
 3979 
 3980 //----------FRAME--------------------------------------------------------------
 3981 // Definition of frame structure and management information.
 3982 //
 3983 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3984 //                             |   (to get allocators register number
 3985 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3986 //  r   CALLER     |        |
 3987 //  o     |        +--------+      pad to even-align allocators stack-slot
 3988 //  w     V        |  pad0  |        numbers; owned by CALLER
 3989 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3990 //  h     ^        |   in   |  5
 3991 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3992 //  |     |        |        |  3
 3993 //  |     |        +--------+
 3994 //  V     |        | old out|      Empty on Intel, window on Sparc
 3995 //        |    old |preserve|      Must be even aligned.
 3996 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3997 //        |        |   in   |  3   area for Intel ret address
 3998 //     Owned by    |preserve|      Empty on Sparc.
 3999 //       SELF      +--------+
 4000 //        |        |  pad2  |  2   pad to align old SP
 4001 //        |        +--------+  1
 4002 //        |        | locks  |  0
 4003 //        |        +--------+----> OptoReg::stack0(), even aligned
 4004 //        |        |  pad1  | 11   pad to align new SP
 4005 //        |        +--------+
 4006 //        |        |        | 10
 4007 //        |        | spills |  9   spills
 4008 //        V        |        |  8   (pad0 slot for callee)
 4009 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4010 //        ^        |  out   |  7
 4011 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4012 //     Owned by    +--------+
 4013 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4014 //        |    new |preserve|      Must be even-aligned.
 4015 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4016 //        |        |        |
 4017 //
 4018 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4019 //         known from SELF's arguments and the Java calling convention.
 4020 //         Region 6-7 is determined per call site.
 4021 // Note 2: If the calling convention leaves holes in the incoming argument
 4022 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4023 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4024 //         incoming area, as the Java calling convention is completely under
 4025 //         the control of the AD file.  Doubles can be sorted and packed to
 4026 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4027 //         varargs C calling conventions.
 4028 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4029 //         even aligned with pad0 as needed.
 4030 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4031 //           (the latter is true on Intel but is it false on AArch64?)
 4032 //         region 6-11 is even aligned; it may be padded out more so that
 4033 //         the region from SP to FP meets the minimum stack alignment.
 4034 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4035 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4036 //         SP meets the minimum alignment.
 4037 
 4038 frame %{
 4039   // These three registers define part of the calling convention
 4040   // between compiled code and the interpreter.
 4041 
 4042   // Inline Cache Register or Method for I2C.
 4043   inline_cache_reg(R12);
 4044 
 4045   // Number of stack slots consumed by locking an object
 4046   sync_stack_slots(2);
 4047 
 4048   // Compiled code's Frame Pointer
 4049   frame_pointer(R31);
 4050 
 4051   // Interpreter stores its frame pointer in a register which is
 4052   // stored to the stack by I2CAdaptors.
 4053   // I2CAdaptors convert from interpreted java to compiled java.
 4054   interpreter_frame_pointer(R29);
 4055 
 4056   // Stack alignment requirement
 4057   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4058 
 4059   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4060   // for calls to C.  Supports the var-args backing area for register parms.
 4061   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4062 
 4063   // The after-PROLOG location of the return address.  Location of
 4064   // return address specifies a type (REG or STACK) and a number
 4065   // representing the register number (i.e. - use a register name) or
 4066   // stack slot.
 4067   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4068   // Otherwise, it is above the locks and verification slot and alignment word
 4069   // TODO this may well be correct but need to check why that - 2 is there
 4070   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4071   // which folds in the space used for monitors
 4072   return_addr(STACK - 2 +
 4073               align_up((Compile::current()->in_preserve_stack_slots() +
 4074                         Compile::current()->fixed_slots()),
 4075                        stack_alignment_in_slots()));
 4076 
 4077   // Location of compiled Java return values.  Same as C for now.
 4078   return_value
 4079   %{
 4080     // TODO do we allow ideal_reg == Op_RegN???
 4081     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4082            "only return normal values");
 4083 
 4084     static const int lo[Op_RegL + 1] = { // enum name
 4085       0,                                 // Op_Node
 4086       0,                                 // Op_Set
 4087       R0_num,                            // Op_RegN
 4088       R0_num,                            // Op_RegI
 4089       R0_num,                            // Op_RegP
 4090       V0_num,                            // Op_RegF
 4091       V0_num,                            // Op_RegD
 4092       R0_num                             // Op_RegL
 4093     };
 4094 
 4095     static const int hi[Op_RegL + 1] = { // enum name
 4096       0,                                 // Op_Node
 4097       0,                                 // Op_Set
 4098       OptoReg::Bad,                      // Op_RegN
 4099       OptoReg::Bad,                      // Op_RegI
 4100       R0_H_num,                          // Op_RegP
 4101       OptoReg::Bad,                      // Op_RegF
 4102       V0_H_num,                          // Op_RegD
 4103       R0_H_num                           // Op_RegL
 4104     };
 4105 
 4106     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4107   %}
 4108 %}
 4109 
 4110 //----------ATTRIBUTES---------------------------------------------------------
 4111 //----------Operand Attributes-------------------------------------------------
 4112 op_attrib op_cost(1);        // Required cost attribute
 4113 
 4114 //----------Instruction Attributes---------------------------------------------
 4115 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4116 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4117 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4118                                 // a non-matching short branch variant
 4119                                 // of some long branch?
 4120 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4121                                 // be a power of 2) specifies the
 4122                                 // alignment that some part of the
 4123                                 // instruction (not necessarily the
 4124                                 // start) requires.  If > 1, a
 4125                                 // compute_padding() function must be
 4126                                 // provided for the instruction
 4127 
 4128 //----------OPERANDS-----------------------------------------------------------
 4129 // Operand definitions must precede instruction definitions for correct parsing
 4130 // in the ADLC because operands constitute user defined types which are used in
 4131 // instruction definitions.
 4132 
 4133 //----------Simple Operands----------------------------------------------------
 4134 
 4135 // Integer operands 32 bit
 4136 // 32 bit immediate
 4137 operand immI()
 4138 %{
 4139   match(ConI);
 4140 
 4141   op_cost(0);
 4142   format %{ %}
 4143   interface(CONST_INTER);
 4144 %}
 4145 
 4146 // 32 bit zero
 4147 operand immI0()
 4148 %{
 4149   predicate(n->get_int() == 0);
 4150   match(ConI);
 4151 
 4152   op_cost(0);
 4153   format %{ %}
 4154   interface(CONST_INTER);
 4155 %}
 4156 
 4157 // 32 bit unit increment
 4158 operand immI_1()
 4159 %{
 4160   predicate(n->get_int() == 1);
 4161   match(ConI);
 4162 
 4163   op_cost(0);
 4164   format %{ %}
 4165   interface(CONST_INTER);
 4166 %}
 4167 
 4168 // 32 bit unit decrement
 4169 operand immI_M1()
 4170 %{
 4171   predicate(n->get_int() == -1);
 4172   match(ConI);
 4173 
 4174   op_cost(0);
 4175   format %{ %}
 4176   interface(CONST_INTER);
 4177 %}
 4178 
 4179 // Shift values for add/sub extension shift
 4180 operand immIExt()
 4181 %{
 4182   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4183   match(ConI);
 4184 
 4185   op_cost(0);
 4186   format %{ %}
 4187   interface(CONST_INTER);
 4188 %}
 4189 
 4190 operand immI_gt_1()
 4191 %{
 4192   predicate(n->get_int() > 1);
 4193   match(ConI);
 4194 
 4195   op_cost(0);
 4196   format %{ %}
 4197   interface(CONST_INTER);
 4198 %}
 4199 
 4200 operand immI_le_4()
 4201 %{
 4202   predicate(n->get_int() <= 4);
 4203   match(ConI);
 4204 
 4205   op_cost(0);
 4206   format %{ %}
 4207   interface(CONST_INTER);
 4208 %}
 4209 
 4210 operand immI_31()
 4211 %{
 4212   predicate(n->get_int() == 31);
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immI_2()
 4221 %{
 4222   predicate(n->get_int() == 2);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 operand immI_4()
 4231 %{
 4232   predicate(n->get_int() == 4);
 4233   match(ConI);
 4234 
 4235   op_cost(0);
 4236   format %{ %}
 4237   interface(CONST_INTER);
 4238 %}
 4239 
 4240 operand immI_8()
 4241 %{
 4242   predicate(n->get_int() == 8);
 4243   match(ConI);
 4244 
 4245   op_cost(0);
 4246   format %{ %}
 4247   interface(CONST_INTER);
 4248 %}
 4249 
 4250 operand immI_16()
 4251 %{
 4252   predicate(n->get_int() == 16);
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 operand immI_24()
 4261 %{
 4262   predicate(n->get_int() == 24);
 4263   match(ConI);
 4264 
 4265   op_cost(0);
 4266   format %{ %}
 4267   interface(CONST_INTER);
 4268 %}
 4269 
 4270 operand immI_32()
 4271 %{
 4272   predicate(n->get_int() == 32);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 operand immI_48()
 4281 %{
 4282   predicate(n->get_int() == 48);
 4283   match(ConI);
 4284 
 4285   op_cost(0);
 4286   format %{ %}
 4287   interface(CONST_INTER);
 4288 %}
 4289 
 4290 operand immI_56()
 4291 %{
 4292   predicate(n->get_int() == 56);
 4293   match(ConI);
 4294 
 4295   op_cost(0);
 4296   format %{ %}
 4297   interface(CONST_INTER);
 4298 %}
 4299 
 4300 operand immI_63()
 4301 %{
 4302   predicate(n->get_int() == 63);
 4303   match(ConI);
 4304 
 4305   op_cost(0);
 4306   format %{ %}
 4307   interface(CONST_INTER);
 4308 %}
 4309 
 4310 operand immI_64()
 4311 %{
 4312   predicate(n->get_int() == 64);
 4313   match(ConI);
 4314 
 4315   op_cost(0);
 4316   format %{ %}
 4317   interface(CONST_INTER);
 4318 %}
 4319 
 4320 operand immI_255()
 4321 %{
 4322   predicate(n->get_int() == 255);
 4323   match(ConI);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 operand immI_65535()
 4331 %{
 4332   predicate(n->get_int() == 65535);
 4333   match(ConI);
 4334 
 4335   op_cost(0);
 4336   format %{ %}
 4337   interface(CONST_INTER);
 4338 %}
 4339 
 4340 operand immL_255()
 4341 %{
 4342   predicate(n->get_long() == 255L);
 4343   match(ConL);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 operand immL_65535()
 4351 %{
 4352   predicate(n->get_long() == 65535L);
 4353   match(ConL);
 4354 
 4355   op_cost(0);
 4356   format %{ %}
 4357   interface(CONST_INTER);
 4358 %}
 4359 
 4360 operand immL_4294967295()
 4361 %{
 4362   predicate(n->get_long() == 4294967295L);
 4363   match(ConL);
 4364 
 4365   op_cost(0);
 4366   format %{ %}
 4367   interface(CONST_INTER);
 4368 %}
 4369 
 4370 operand immL_bitmask()
 4371 %{
 4372   predicate((n->get_long() != 0)
 4373             && ((n->get_long() & 0xc000000000000000l) == 0)
 4374             && is_power_of_2(n->get_long() + 1));
 4375   match(ConL);
 4376 
 4377   op_cost(0);
 4378   format %{ %}
 4379   interface(CONST_INTER);
 4380 %}
 4381 
 4382 operand immI_bitmask()
 4383 %{
 4384   predicate((n->get_int() != 0)
 4385             && ((n->get_int() & 0xc0000000) == 0)
 4386             && is_power_of_2(n->get_int() + 1));
 4387   match(ConI);
 4388 
 4389   op_cost(0);
 4390   format %{ %}
 4391   interface(CONST_INTER);
 4392 %}
 4393 
 4394 operand immL_positive_bitmaskI()
 4395 %{
 4396   predicate((n->get_long() != 0)
 4397             && ((julong)n->get_long() < 0x80000000ULL)
 4398             && is_power_of_2(n->get_long() + 1));
 4399   match(ConL);
 4400 
 4401   op_cost(0);
 4402   format %{ %}
 4403   interface(CONST_INTER);
 4404 %}
 4405 
 4406 // Scale values for scaled offset addressing modes (up to long but not quad)
 4407 operand immIScale()
 4408 %{
 4409   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4410   match(ConI);
 4411 
 4412   op_cost(0);
 4413   format %{ %}
 4414   interface(CONST_INTER);
 4415 %}
 4416 
 4417 // 26 bit signed offset -- for pc-relative branches
 4418 operand immI26()
 4419 %{
 4420   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4421   match(ConI);
 4422 
 4423   op_cost(0);
 4424   format %{ %}
 4425   interface(CONST_INTER);
 4426 %}
 4427 
 4428 // 19 bit signed offset -- for pc-relative loads
 4429 operand immI19()
 4430 %{
 4431   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 // 12 bit unsigned offset -- for base plus immediate loads
 4440 operand immIU12()
 4441 %{
 4442   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4443   match(ConI);
 4444 
 4445   op_cost(0);
 4446   format %{ %}
 4447   interface(CONST_INTER);
 4448 %}
 4449 
 4450 operand immLU12()
 4451 %{
 4452   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4453   match(ConL);
 4454 
 4455   op_cost(0);
 4456   format %{ %}
 4457   interface(CONST_INTER);
 4458 %}
 4459 
 4460 // Offset for scaled or unscaled immediate loads and stores
 4461 operand immIOffset()
 4462 %{
 4463   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4464   match(ConI);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 operand immIOffset1()
 4472 %{
 4473   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4474   match(ConI);
 4475 
 4476   op_cost(0);
 4477   format %{ %}
 4478   interface(CONST_INTER);
 4479 %}
 4480 
 4481 operand immIOffset2()
 4482 %{
 4483   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4484   match(ConI);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immIOffset4()
 4492 %{
 4493   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4494   match(ConI);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 operand immIOffset8()
 4502 %{
 4503   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4504   match(ConI);
 4505 
 4506   op_cost(0);
 4507   format %{ %}
 4508   interface(CONST_INTER);
 4509 %}
 4510 
 4511 operand immIOffset16()
 4512 %{
 4513   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4514   match(ConI);
 4515 
 4516   op_cost(0);
 4517   format %{ %}
 4518   interface(CONST_INTER);
 4519 %}
 4520 
 4521 operand immLoffset()
 4522 %{
 4523   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4524   match(ConL);
 4525 
 4526   op_cost(0);
 4527   format %{ %}
 4528   interface(CONST_INTER);
 4529 %}
 4530 
 4531 operand immLoffset1()
 4532 %{
 4533   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4534   match(ConL);
 4535 
 4536   op_cost(0);
 4537   format %{ %}
 4538   interface(CONST_INTER);
 4539 %}
 4540 
 4541 operand immLoffset2()
 4542 %{
 4543   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4544   match(ConL);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 operand immLoffset4()
 4552 %{
 4553   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4554   match(ConL);
 4555 
 4556   op_cost(0);
 4557   format %{ %}
 4558   interface(CONST_INTER);
 4559 %}
 4560 
 4561 operand immLoffset8()
 4562 %{
 4563   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4564   match(ConL);
 4565 
 4566   op_cost(0);
 4567   format %{ %}
 4568   interface(CONST_INTER);
 4569 %}
 4570 
 4571 operand immLoffset16()
 4572 %{
 4573   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4574   match(ConL);
 4575 
 4576   op_cost(0);
 4577   format %{ %}
 4578   interface(CONST_INTER);
 4579 %}
 4580 
 4581 // 8 bit signed value.
 4582 operand immI8()
 4583 %{
 4584   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4585   match(ConI);
 4586 
 4587   op_cost(0);
 4588   format %{ %}
 4589   interface(CONST_INTER);
 4590 %}
 4591 
 4592 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4593 operand immI8_shift8()
 4594 %{
 4595   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4596             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4597   match(ConI);
 4598 
 4599   op_cost(0);
 4600   format %{ %}
 4601   interface(CONST_INTER);
 4602 %}
 4603 
 4604 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4605 operand immL8_shift8()
 4606 %{
 4607   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4608             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4609   match(ConL);
 4610 
 4611   op_cost(0);
 4612   format %{ %}
 4613   interface(CONST_INTER);
 4614 %}
 4615 
 4616 // 32 bit integer valid for add sub immediate
 4617 operand immIAddSub()
 4618 %{
 4619   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4620   match(ConI);
 4621   op_cost(0);
 4622   format %{ %}
 4623   interface(CONST_INTER);
 4624 %}
 4625 
 4626 // 32 bit unsigned integer valid for logical immediate
 4627 // TODO -- check this is right when e.g the mask is 0x80000000
 4628 operand immILog()
 4629 %{
 4630   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4631   match(ConI);
 4632 
 4633   op_cost(0);
 4634   format %{ %}
 4635   interface(CONST_INTER);
 4636 %}
 4637 
 4638 // Integer operands 64 bit
 4639 // 64 bit immediate
 4640 operand immL()
 4641 %{
 4642   match(ConL);
 4643 
 4644   op_cost(0);
 4645   format %{ %}
 4646   interface(CONST_INTER);
 4647 %}
 4648 
 4649 // 64 bit zero
 4650 operand immL0()
 4651 %{
 4652   predicate(n->get_long() == 0);
 4653   match(ConL);
 4654 
 4655   op_cost(0);
 4656   format %{ %}
 4657   interface(CONST_INTER);
 4658 %}
 4659 
 4660 // 64 bit unit increment
 4661 operand immL_1()
 4662 %{
 4663   predicate(n->get_long() == 1);
 4664   match(ConL);
 4665 
 4666   op_cost(0);
 4667   format %{ %}
 4668   interface(CONST_INTER);
 4669 %}
 4670 
 4671 // 64 bit unit decrement
 4672 operand immL_M1()
 4673 %{
 4674   predicate(n->get_long() == -1);
 4675   match(ConL);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 // 32 bit offset of pc in thread anchor
 4683 
 4684 operand immL_pc_off()
 4685 %{
 4686   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4687                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4688   match(ConL);
 4689 
 4690   op_cost(0);
 4691   format %{ %}
 4692   interface(CONST_INTER);
 4693 %}
 4694 
 4695 // 64 bit integer valid for add sub immediate
 4696 operand immLAddSub()
 4697 %{
 4698   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4699   match(ConL);
 4700   op_cost(0);
 4701   format %{ %}
 4702   interface(CONST_INTER);
 4703 %}
 4704 
 4705 // 64 bit integer valid for logical immediate
 4706 operand immLLog()
 4707 %{
 4708   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4709   match(ConL);
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(CONST_INTER);
 4713 %}
 4714 
 4715 // Long Immediate: low 32-bit mask
 4716 operand immL_32bits()
 4717 %{
 4718   predicate(n->get_long() == 0xFFFFFFFFL);
 4719   match(ConL);
 4720   op_cost(0);
 4721   format %{ %}
 4722   interface(CONST_INTER);
 4723 %}
 4724 
 4725 // Pointer operands
 4726 // Pointer Immediate
 4727 operand immP()
 4728 %{
 4729   match(ConP);
 4730 
 4731   op_cost(0);
 4732   format %{ %}
 4733   interface(CONST_INTER);
 4734 %}
 4735 
 4736 // NULL Pointer Immediate
 4737 operand immP0()
 4738 %{
 4739   predicate(n->get_ptr() == 0);
 4740   match(ConP);
 4741 
 4742   op_cost(0);
 4743   format %{ %}
 4744   interface(CONST_INTER);
 4745 %}
 4746 
 4747 // Pointer Immediate One
 4748 // this is used in object initialization (initial object header)
 4749 operand immP_1()
 4750 %{
 4751   predicate(n->get_ptr() == 1);
 4752   match(ConP);
 4753 
 4754   op_cost(0);
 4755   format %{ %}
 4756   interface(CONST_INTER);
 4757 %}
 4758 
 4759 // Card Table Byte Map Base
 4760 operand immByteMapBase()
 4761 %{
 4762   // Get base of card map
 4763   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4764             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4765   match(ConP);
 4766 
 4767   op_cost(0);
 4768   format %{ %}
 4769   interface(CONST_INTER);
 4770 %}
 4771 
 4772 // Pointer Immediate Minus One
 4773 // this is used when we want to write the current PC to the thread anchor
 4774 operand immP_M1()
 4775 %{
 4776   predicate(n->get_ptr() == -1);
 4777   match(ConP);
 4778 
 4779   op_cost(0);
 4780   format %{ %}
 4781   interface(CONST_INTER);
 4782 %}
 4783 
 4784 // Pointer Immediate Minus Two
 4785 // this is used when we want to write the current PC to the thread anchor
 4786 operand immP_M2()
 4787 %{
 4788   predicate(n->get_ptr() == -2);
 4789   match(ConP);
 4790 
 4791   op_cost(0);
 4792   format %{ %}
 4793   interface(CONST_INTER);
 4794 %}
 4795 
 4796 // Float and Double operands
 4797 // Double Immediate
 4798 operand immD()
 4799 %{
 4800   match(ConD);
 4801   op_cost(0);
 4802   format %{ %}
 4803   interface(CONST_INTER);
 4804 %}
 4805 
 4806 // Double Immediate: +0.0d
 4807 operand immD0()
 4808 %{
 4809   predicate(jlong_cast(n->getd()) == 0);
 4810   match(ConD);
 4811 
 4812   op_cost(0);
 4813   format %{ %}
 4814   interface(CONST_INTER);
 4815 %}
 4816 
 4817 // constant 'double +0.0'.
 4818 operand immDPacked()
 4819 %{
 4820   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4821   match(ConD);
 4822   op_cost(0);
 4823   format %{ %}
 4824   interface(CONST_INTER);
 4825 %}
 4826 
 4827 // Float Immediate
 4828 operand immF()
 4829 %{
 4830   match(ConF);
 4831   op_cost(0);
 4832   format %{ %}
 4833   interface(CONST_INTER);
 4834 %}
 4835 
 4836 // Float Immediate: +0.0f.
 4837 operand immF0()
 4838 %{
 4839   predicate(jint_cast(n->getf()) == 0);
 4840   match(ConF);
 4841 
 4842   op_cost(0);
 4843   format %{ %}
 4844   interface(CONST_INTER);
 4845 %}
 4846 
 4847 //
 4848 operand immFPacked()
 4849 %{
 4850   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4851   match(ConF);
 4852   op_cost(0);
 4853   format %{ %}
 4854   interface(CONST_INTER);
 4855 %}
 4856 
 4857 // Narrow pointer operands
 4858 // Narrow Pointer Immediate
 4859 operand immN()
 4860 %{
 4861   match(ConN);
 4862 
 4863   op_cost(0);
 4864   format %{ %}
 4865   interface(CONST_INTER);
 4866 %}
 4867 
 4868 // Narrow NULL Pointer Immediate
 4869 operand immN0()
 4870 %{
 4871   predicate(n->get_narrowcon() == 0);
 4872   match(ConN);
 4873 
 4874   op_cost(0);
 4875   format %{ %}
 4876   interface(CONST_INTER);
 4877 %}
 4878 
 4879 operand immNKlass()
 4880 %{
 4881   match(ConNKlass);
 4882 
 4883   op_cost(0);
 4884   format %{ %}
 4885   interface(CONST_INTER);
 4886 %}
 4887 
 4888 // Integer 32 bit Register Operands
 4889 // Integer 32 bitRegister (excludes SP)
 4890 operand iRegI()
 4891 %{
 4892   constraint(ALLOC_IN_RC(any_reg32));
 4893   match(RegI);
 4894   match(iRegINoSp);
 4895   op_cost(0);
 4896   format %{ %}
 4897   interface(REG_INTER);
 4898 %}
 4899 
 4900 // Integer 32 bit Register not Special
 4901 operand iRegINoSp()
 4902 %{
 4903   constraint(ALLOC_IN_RC(no_special_reg32));
 4904   match(RegI);
 4905   op_cost(0);
 4906   format %{ %}
 4907   interface(REG_INTER);
 4908 %}
 4909 
 4910 // Integer 64 bit Register Operands
 4911 // Integer 64 bit Register (includes SP)
 4912 operand iRegL()
 4913 %{
 4914   constraint(ALLOC_IN_RC(any_reg));
 4915   match(RegL);
 4916   match(iRegLNoSp);
 4917   op_cost(0);
 4918   format %{ %}
 4919   interface(REG_INTER);
 4920 %}
 4921 
 4922 // Integer 64 bit Register not Special
 4923 operand iRegLNoSp()
 4924 %{
 4925   constraint(ALLOC_IN_RC(no_special_reg));
 4926   match(RegL);
 4927   match(iRegL_R0);
 4928   format %{ %}
 4929   interface(REG_INTER);
 4930 %}
 4931 
 4932 // Pointer Register Operands
 4933 // Pointer Register
 4934 operand iRegP()
 4935 %{
 4936   constraint(ALLOC_IN_RC(ptr_reg));
 4937   match(RegP);
 4938   match(iRegPNoSp);
 4939   match(iRegP_R0);
 4940   //match(iRegP_R2);
 4941   //match(iRegP_R4);
 4942   //match(iRegP_R5);
 4943   match(thread_RegP);
 4944   op_cost(0);
 4945   format %{ %}
 4946   interface(REG_INTER);
 4947 %}
 4948 
 4949 // Pointer 64 bit Register not Special
 4950 operand iRegPNoSp()
 4951 %{
 4952   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4953   match(RegP);
 4954   // match(iRegP);
 4955   // match(iRegP_R0);
 4956   // match(iRegP_R2);
 4957   // match(iRegP_R4);
 4958   // match(iRegP_R5);
 4959   // match(thread_RegP);
 4960   op_cost(0);
 4961   format %{ %}
 4962   interface(REG_INTER);
 4963 %}
 4964 
 4965 // Pointer 64 bit Register R0 only
 4966 operand iRegP_R0()
 4967 %{
 4968   constraint(ALLOC_IN_RC(r0_reg));
 4969   match(RegP);
 4970   // match(iRegP);
 4971   match(iRegPNoSp);
 4972   op_cost(0);
 4973   format %{ %}
 4974   interface(REG_INTER);
 4975 %}
 4976 
 4977 // Pointer 64 bit Register R1 only
 4978 operand iRegP_R1()
 4979 %{
 4980   constraint(ALLOC_IN_RC(r1_reg));
 4981   match(RegP);
 4982   // match(iRegP);
 4983   match(iRegPNoSp);
 4984   op_cost(0);
 4985   format %{ %}
 4986   interface(REG_INTER);
 4987 %}
 4988 
 4989 // Pointer 64 bit Register R2 only
 4990 operand iRegP_R2()
 4991 %{
 4992   constraint(ALLOC_IN_RC(r2_reg));
 4993   match(RegP);
 4994   // match(iRegP);
 4995   match(iRegPNoSp);
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(REG_INTER);
 4999 %}
 5000 
 5001 // Pointer 64 bit Register R3 only
 5002 operand iRegP_R3()
 5003 %{
 5004   constraint(ALLOC_IN_RC(r3_reg));
 5005   match(RegP);
 5006   // match(iRegP);
 5007   match(iRegPNoSp);
 5008   op_cost(0);
 5009   format %{ %}
 5010   interface(REG_INTER);
 5011 %}
 5012 
 5013 // Pointer 64 bit Register R4 only
 5014 operand iRegP_R4()
 5015 %{
 5016   constraint(ALLOC_IN_RC(r4_reg));
 5017   match(RegP);
 5018   // match(iRegP);
 5019   match(iRegPNoSp);
 5020   op_cost(0);
 5021   format %{ %}
 5022   interface(REG_INTER);
 5023 %}
 5024 
 5025 // Pointer 64 bit Register R5 only
 5026 operand iRegP_R5()
 5027 %{
 5028   constraint(ALLOC_IN_RC(r5_reg));
 5029   match(RegP);
 5030   // match(iRegP);
 5031   match(iRegPNoSp);
 5032   op_cost(0);
 5033   format %{ %}
 5034   interface(REG_INTER);
 5035 %}
 5036 
 5037 // Pointer 64 bit Register R10 only
 5038 operand iRegP_R10()
 5039 %{
 5040   constraint(ALLOC_IN_RC(r10_reg));
 5041   match(RegP);
 5042   // match(iRegP);
 5043   match(iRegPNoSp);
 5044   op_cost(0);
 5045   format %{ %}
 5046   interface(REG_INTER);
 5047 %}
 5048 
 5049 // Long 64 bit Register R0 only
 5050 operand iRegL_R0()
 5051 %{
 5052   constraint(ALLOC_IN_RC(r0_reg));
 5053   match(RegL);
 5054   match(iRegLNoSp);
 5055   op_cost(0);
 5056   format %{ %}
 5057   interface(REG_INTER);
 5058 %}
 5059 
 5060 // Long 64 bit Register R2 only
 5061 operand iRegL_R2()
 5062 %{
 5063   constraint(ALLOC_IN_RC(r2_reg));
 5064   match(RegL);
 5065   match(iRegLNoSp);
 5066   op_cost(0);
 5067   format %{ %}
 5068   interface(REG_INTER);
 5069 %}
 5070 
 5071 // Long 64 bit Register R3 only
 5072 operand iRegL_R3()
 5073 %{
 5074   constraint(ALLOC_IN_RC(r3_reg));
 5075   match(RegL);
 5076   match(iRegLNoSp);
 5077   op_cost(0);
 5078   format %{ %}
 5079   interface(REG_INTER);
 5080 %}
 5081 
 5082 // Long 64 bit Register R11 only
 5083 operand iRegL_R11()
 5084 %{
 5085   constraint(ALLOC_IN_RC(r11_reg));
 5086   match(RegL);
 5087   match(iRegLNoSp);
 5088   op_cost(0);
 5089   format %{ %}
 5090   interface(REG_INTER);
 5091 %}
 5092 
 5093 // Pointer 64 bit Register FP only
 5094 operand iRegP_FP()
 5095 %{
 5096   constraint(ALLOC_IN_RC(fp_reg));
 5097   match(RegP);
 5098   // match(iRegP);
 5099   op_cost(0);
 5100   format %{ %}
 5101   interface(REG_INTER);
 5102 %}
 5103 
 5104 // Register R0 only
 5105 operand iRegI_R0()
 5106 %{
 5107   constraint(ALLOC_IN_RC(int_r0_reg));
 5108   match(RegI);
 5109   match(iRegINoSp);
 5110   op_cost(0);
 5111   format %{ %}
 5112   interface(REG_INTER);
 5113 %}
 5114 
 5115 // Register R2 only
 5116 operand iRegI_R2()
 5117 %{
 5118   constraint(ALLOC_IN_RC(int_r2_reg));
 5119   match(RegI);
 5120   match(iRegINoSp);
 5121   op_cost(0);
 5122   format %{ %}
 5123   interface(REG_INTER);
 5124 %}
 5125 
 5126 // Register R3 only
 5127 operand iRegI_R3()
 5128 %{
 5129   constraint(ALLOC_IN_RC(int_r3_reg));
 5130   match(RegI);
 5131   match(iRegINoSp);
 5132   op_cost(0);
 5133   format %{ %}
 5134   interface(REG_INTER);
 5135 %}
 5136 
 5137 
 5138 // Register R4 only
 5139 operand iRegI_R4()
 5140 %{
 5141   constraint(ALLOC_IN_RC(int_r4_reg));
 5142   match(RegI);
 5143   match(iRegINoSp);
 5144   op_cost(0);
 5145   format %{ %}
 5146   interface(REG_INTER);
 5147 %}
 5148 
 5149 
 5150 // Pointer Register Operands
 5151 // Narrow Pointer Register
 5152 operand iRegN()
 5153 %{
 5154   constraint(ALLOC_IN_RC(any_reg32));
 5155   match(RegN);
 5156   match(iRegNNoSp);
 5157   op_cost(0);
 5158   format %{ %}
 5159   interface(REG_INTER);
 5160 %}
 5161 
 5162 operand iRegN_R0()
 5163 %{
 5164   constraint(ALLOC_IN_RC(r0_reg));
 5165   match(iRegN);
 5166   op_cost(0);
 5167   format %{ %}
 5168   interface(REG_INTER);
 5169 %}
 5170 
 5171 operand iRegN_R2()
 5172 %{
 5173   constraint(ALLOC_IN_RC(r2_reg));
 5174   match(iRegN);
 5175   op_cost(0);
 5176   format %{ %}
 5177   interface(REG_INTER);
 5178 %}
 5179 
 5180 operand iRegN_R3()
 5181 %{
 5182   constraint(ALLOC_IN_RC(r3_reg));
 5183   match(iRegN);
 5184   op_cost(0);
 5185   format %{ %}
 5186   interface(REG_INTER);
 5187 %}
 5188 
 5189 // Integer 64 bit Register not Special
 5190 operand iRegNNoSp()
 5191 %{
 5192   constraint(ALLOC_IN_RC(no_special_reg32));
 5193   match(RegN);
 5194   op_cost(0);
 5195   format %{ %}
 5196   interface(REG_INTER);
 5197 %}
 5198 
 5199 // heap base register -- used for encoding immN0
 5200 
 5201 operand iRegIHeapbase()
 5202 %{
 5203   constraint(ALLOC_IN_RC(heapbase_reg));
 5204   match(RegI);
 5205   op_cost(0);
 5206   format %{ %}
 5207   interface(REG_INTER);
 5208 %}
 5209 
 5210 // Float Register
 5211 // Float register operands
 5212 operand vRegF()
 5213 %{
 5214   constraint(ALLOC_IN_RC(float_reg));
 5215   match(RegF);
 5216 
 5217   op_cost(0);
 5218   format %{ %}
 5219   interface(REG_INTER);
 5220 %}
 5221 
 5222 // Double Register
 5223 // Double register operands
 5224 operand vRegD()
 5225 %{
 5226   constraint(ALLOC_IN_RC(double_reg));
 5227   match(RegD);
 5228 
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 // Generic vector class. This will be used for
 5235 // all vector operands, including NEON and SVE,
 5236 // but currently only used for SVE VecA.
 5237 operand vReg()
 5238 %{
 5239   constraint(ALLOC_IN_RC(vectora_reg));
 5240   match(VecA);
 5241   op_cost(0);
 5242   format %{ %}
 5243   interface(REG_INTER);
 5244 %}
 5245 
 5246 operand vecD()
 5247 %{
 5248   constraint(ALLOC_IN_RC(vectord_reg));
 5249   match(VecD);
 5250 
 5251   op_cost(0);
 5252   format %{ %}
 5253   interface(REG_INTER);
 5254 %}
 5255 
 5256 operand vecX()
 5257 %{
 5258   constraint(ALLOC_IN_RC(vectorx_reg));
 5259   match(VecX);
 5260 
 5261   op_cost(0);
 5262   format %{ %}
 5263   interface(REG_INTER);
 5264 %}
 5265 
 5266 operand vRegD_V0()
 5267 %{
 5268   constraint(ALLOC_IN_RC(v0_reg));
 5269   match(RegD);
 5270   op_cost(0);
 5271   format %{ %}
 5272   interface(REG_INTER);
 5273 %}
 5274 
 5275 operand vRegD_V1()
 5276 %{
 5277   constraint(ALLOC_IN_RC(v1_reg));
 5278   match(RegD);
 5279   op_cost(0);
 5280   format %{ %}
 5281   interface(REG_INTER);
 5282 %}
 5283 
 5284 operand vRegD_V2()
 5285 %{
 5286   constraint(ALLOC_IN_RC(v2_reg));
 5287   match(RegD);
 5288   op_cost(0);
 5289   format %{ %}
 5290   interface(REG_INTER);
 5291 %}
 5292 
 5293 operand vRegD_V3()
 5294 %{
 5295   constraint(ALLOC_IN_RC(v3_reg));
 5296   match(RegD);
 5297   op_cost(0);
 5298   format %{ %}
 5299   interface(REG_INTER);
 5300 %}
 5301 
 5302 operand vRegD_V4()
 5303 %{
 5304   constraint(ALLOC_IN_RC(v4_reg));
 5305   match(RegD);
 5306   op_cost(0);
 5307   format %{ %}
 5308   interface(REG_INTER);
 5309 %}
 5310 
 5311 operand vRegD_V5()
 5312 %{
 5313   constraint(ALLOC_IN_RC(v5_reg));
 5314   match(RegD);
 5315   op_cost(0);
 5316   format %{ %}
 5317   interface(REG_INTER);
 5318 %}
 5319 
 5320 operand vRegD_V6()
 5321 %{
 5322   constraint(ALLOC_IN_RC(v6_reg));
 5323   match(RegD);
 5324   op_cost(0);
 5325   format %{ %}
 5326   interface(REG_INTER);
 5327 %}
 5328 
 5329 operand vRegD_V7()
 5330 %{
 5331   constraint(ALLOC_IN_RC(v7_reg));
 5332   match(RegD);
 5333   op_cost(0);
 5334   format %{ %}
 5335   interface(REG_INTER);
 5336 %}
 5337 
 5338 operand vRegD_V8()
 5339 %{
 5340   constraint(ALLOC_IN_RC(v8_reg));
 5341   match(RegD);
 5342   op_cost(0);
 5343   format %{ %}
 5344   interface(REG_INTER);
 5345 %}
 5346 
 5347 operand vRegD_V9()
 5348 %{
 5349   constraint(ALLOC_IN_RC(v9_reg));
 5350   match(RegD);
 5351   op_cost(0);
 5352   format %{ %}
 5353   interface(REG_INTER);
 5354 %}
 5355 
 5356 operand vRegD_V10()
 5357 %{
 5358   constraint(ALLOC_IN_RC(v10_reg));
 5359   match(RegD);
 5360   op_cost(0);
 5361   format %{ %}
 5362   interface(REG_INTER);
 5363 %}
 5364 
 5365 operand vRegD_V11()
 5366 %{
 5367   constraint(ALLOC_IN_RC(v11_reg));
 5368   match(RegD);
 5369   op_cost(0);
 5370   format %{ %}
 5371   interface(REG_INTER);
 5372 %}
 5373 
 5374 operand vRegD_V12()
 5375 %{
 5376   constraint(ALLOC_IN_RC(v12_reg));
 5377   match(RegD);
 5378   op_cost(0);
 5379   format %{ %}
 5380   interface(REG_INTER);
 5381 %}
 5382 
 5383 operand vRegD_V13()
 5384 %{
 5385   constraint(ALLOC_IN_RC(v13_reg));
 5386   match(RegD);
 5387   op_cost(0);
 5388   format %{ %}
 5389   interface(REG_INTER);
 5390 %}
 5391 
 5392 operand vRegD_V14()
 5393 %{
 5394   constraint(ALLOC_IN_RC(v14_reg));
 5395   match(RegD);
 5396   op_cost(0);
 5397   format %{ %}
 5398   interface(REG_INTER);
 5399 %}
 5400 
 5401 operand vRegD_V15()
 5402 %{
 5403   constraint(ALLOC_IN_RC(v15_reg));
 5404   match(RegD);
 5405   op_cost(0);
 5406   format %{ %}
 5407   interface(REG_INTER);
 5408 %}
 5409 
 5410 operand vRegD_V16()
 5411 %{
 5412   constraint(ALLOC_IN_RC(v16_reg));
 5413   match(RegD);
 5414   op_cost(0);
 5415   format %{ %}
 5416   interface(REG_INTER);
 5417 %}
 5418 
 5419 operand vRegD_V17()
 5420 %{
 5421   constraint(ALLOC_IN_RC(v17_reg));
 5422   match(RegD);
 5423   op_cost(0);
 5424   format %{ %}
 5425   interface(REG_INTER);
 5426 %}
 5427 
 5428 operand vRegD_V18()
 5429 %{
 5430   constraint(ALLOC_IN_RC(v18_reg));
 5431   match(RegD);
 5432   op_cost(0);
 5433   format %{ %}
 5434   interface(REG_INTER);
 5435 %}
 5436 
 5437 operand vRegD_V19()
 5438 %{
 5439   constraint(ALLOC_IN_RC(v19_reg));
 5440   match(RegD);
 5441   op_cost(0);
 5442   format %{ %}
 5443   interface(REG_INTER);
 5444 %}
 5445 
 5446 operand vRegD_V20()
 5447 %{
 5448   constraint(ALLOC_IN_RC(v20_reg));
 5449   match(RegD);
 5450   op_cost(0);
 5451   format %{ %}
 5452   interface(REG_INTER);
 5453 %}
 5454 
 5455 operand vRegD_V21()
 5456 %{
 5457   constraint(ALLOC_IN_RC(v21_reg));
 5458   match(RegD);
 5459   op_cost(0);
 5460   format %{ %}
 5461   interface(REG_INTER);
 5462 %}
 5463 
 5464 operand vRegD_V22()
 5465 %{
 5466   constraint(ALLOC_IN_RC(v22_reg));
 5467   match(RegD);
 5468   op_cost(0);
 5469   format %{ %}
 5470   interface(REG_INTER);
 5471 %}
 5472 
 5473 operand vRegD_V23()
 5474 %{
 5475   constraint(ALLOC_IN_RC(v23_reg));
 5476   match(RegD);
 5477   op_cost(0);
 5478   format %{ %}
 5479   interface(REG_INTER);
 5480 %}
 5481 
 5482 operand vRegD_V24()
 5483 %{
 5484   constraint(ALLOC_IN_RC(v24_reg));
 5485   match(RegD);
 5486   op_cost(0);
 5487   format %{ %}
 5488   interface(REG_INTER);
 5489 %}
 5490 
 5491 operand vRegD_V25()
 5492 %{
 5493   constraint(ALLOC_IN_RC(v25_reg));
 5494   match(RegD);
 5495   op_cost(0);
 5496   format %{ %}
 5497   interface(REG_INTER);
 5498 %}
 5499 
 5500 operand vRegD_V26()
 5501 %{
 5502   constraint(ALLOC_IN_RC(v26_reg));
 5503   match(RegD);
 5504   op_cost(0);
 5505   format %{ %}
 5506   interface(REG_INTER);
 5507 %}
 5508 
 5509 operand vRegD_V27()
 5510 %{
 5511   constraint(ALLOC_IN_RC(v27_reg));
 5512   match(RegD);
 5513   op_cost(0);
 5514   format %{ %}
 5515   interface(REG_INTER);
 5516 %}
 5517 
 5518 operand vRegD_V28()
 5519 %{
 5520   constraint(ALLOC_IN_RC(v28_reg));
 5521   match(RegD);
 5522   op_cost(0);
 5523   format %{ %}
 5524   interface(REG_INTER);
 5525 %}
 5526 
 5527 operand vRegD_V29()
 5528 %{
 5529   constraint(ALLOC_IN_RC(v29_reg));
 5530   match(RegD);
 5531   op_cost(0);
 5532   format %{ %}
 5533   interface(REG_INTER);
 5534 %}
 5535 
 5536 operand vRegD_V30()
 5537 %{
 5538   constraint(ALLOC_IN_RC(v30_reg));
 5539   match(RegD);
 5540   op_cost(0);
 5541   format %{ %}
 5542   interface(REG_INTER);
 5543 %}
 5544 
 5545 operand vRegD_V31()
 5546 %{
 5547   constraint(ALLOC_IN_RC(v31_reg));
 5548   match(RegD);
 5549   op_cost(0);
 5550   format %{ %}
 5551   interface(REG_INTER);
 5552 %}
 5553 
 5554 operand pReg()
 5555 %{
 5556   constraint(ALLOC_IN_RC(pr_reg));
 5557   match(RegVectMask);
 5558   op_cost(0);
 5559   format %{ %}
 5560   interface(REG_INTER);
 5561 %}
 5562 
 5563 operand pRegGov()
 5564 %{
 5565   constraint(ALLOC_IN_RC(gov_pr));
 5566   match(RegVectMask);
 5567   op_cost(0);
 5568   format %{ %}
 5569   interface(REG_INTER);
 5570 %}
 5571 
 5572 // Flags register, used as output of signed compare instructions
 5573 
 5574 // note that on AArch64 we also use this register as the output for
 5575 // for floating point compare instructions (CmpF CmpD). this ensures
 5576 // that ordered inequality tests use GT, GE, LT or LE none of which
 5577 // pass through cases where the result is unordered i.e. one or both
 5578 // inputs to the compare is a NaN. this means that the ideal code can
 5579 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5580 // (where the comparison should always fail). EQ and NE tests are
 5581 // always generated in ideal code so that unordered folds into the NE
 5582 // case, matching the behaviour of AArch64 NE.
 5583 //
 5584 // This differs from x86 where the outputs of FP compares use a
 5585 // special FP flags registers and where compares based on this
 5586 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5587 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5588 // to explicitly handle the unordered case in branches. x86 also has
 5589 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5590 
 5591 operand rFlagsReg()
 5592 %{
 5593   constraint(ALLOC_IN_RC(int_flags));
 5594   match(RegFlags);
 5595 
 5596   op_cost(0);
 5597   format %{ "RFLAGS" %}
 5598   interface(REG_INTER);
 5599 %}
 5600 
 5601 // Flags register, used as output of unsigned compare instructions
 5602 operand rFlagsRegU()
 5603 %{
 5604   constraint(ALLOC_IN_RC(int_flags));
 5605   match(RegFlags);
 5606 
 5607   op_cost(0);
 5608   format %{ "RFLAGSU" %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 // Special Registers
 5613 
 5614 // Method Register
 5615 operand inline_cache_RegP(iRegP reg)
 5616 %{
 5617   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5618   match(reg);
 5619   match(iRegPNoSp);
 5620   op_cost(0);
 5621   format %{ %}
 5622   interface(REG_INTER);
 5623 %}
 5624 
 5625 // Thread Register
 5626 operand thread_RegP(iRegP reg)
 5627 %{
 5628   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5629   match(reg);
 5630   op_cost(0);
 5631   format %{ %}
 5632   interface(REG_INTER);
 5633 %}
 5634 
 5635 operand lr_RegP(iRegP reg)
 5636 %{
 5637   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5638   match(reg);
 5639   op_cost(0);
 5640   format %{ %}
 5641   interface(REG_INTER);
 5642 %}
 5643 
 5644 //----------Memory Operands----------------------------------------------------
 5645 
 5646 operand indirect(iRegP reg)
 5647 %{
 5648   constraint(ALLOC_IN_RC(ptr_reg));
 5649   match(reg);
 5650   op_cost(0);
 5651   format %{ "[$reg]" %}
 5652   interface(MEMORY_INTER) %{
 5653     base($reg);
 5654     index(0xffffffff);
 5655     scale(0x0);
 5656     disp(0x0);
 5657   %}
 5658 %}
 5659 
 5660 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5661 %{
 5662   constraint(ALLOC_IN_RC(ptr_reg));
 5663   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5664   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5665   op_cost(0);
 5666   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5667   interface(MEMORY_INTER) %{
 5668     base($reg);
 5669     index($ireg);
 5670     scale($scale);
 5671     disp(0x0);
 5672   %}
 5673 %}
 5674 
 5675 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5676 %{
 5677   constraint(ALLOC_IN_RC(ptr_reg));
 5678   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5679   match(AddP reg (LShiftL lreg scale));
 5680   op_cost(0);
 5681   format %{ "$reg, $lreg lsl($scale)" %}
 5682   interface(MEMORY_INTER) %{
 5683     base($reg);
 5684     index($lreg);
 5685     scale($scale);
 5686     disp(0x0);
 5687   %}
 5688 %}
 5689 
 5690 operand indIndexI2L(iRegP reg, iRegI ireg)
 5691 %{
 5692   constraint(ALLOC_IN_RC(ptr_reg));
 5693   match(AddP reg (ConvI2L ireg));
 5694   op_cost(0);
 5695   format %{ "$reg, $ireg, 0, I2L" %}
 5696   interface(MEMORY_INTER) %{
 5697     base($reg);
 5698     index($ireg);
 5699     scale(0x0);
 5700     disp(0x0);
 5701   %}
 5702 %}
 5703 
 5704 operand indIndex(iRegP reg, iRegL lreg)
 5705 %{
 5706   constraint(ALLOC_IN_RC(ptr_reg));
 5707   match(AddP reg lreg);
 5708   op_cost(0);
 5709   format %{ "$reg, $lreg" %}
 5710   interface(MEMORY_INTER) %{
 5711     base($reg);
 5712     index($lreg);
 5713     scale(0x0);
 5714     disp(0x0);
 5715   %}
 5716 %}
 5717 
 5718 operand indOffI(iRegP reg, immIOffset off)
 5719 %{
 5720   constraint(ALLOC_IN_RC(ptr_reg));
 5721   match(AddP reg off);
 5722   op_cost(0);
 5723   format %{ "[$reg, $off]" %}
 5724   interface(MEMORY_INTER) %{
 5725     base($reg);
 5726     index(0xffffffff);
 5727     scale(0x0);
 5728     disp($off);
 5729   %}
 5730 %}
 5731 
 5732 operand indOffI1(iRegP reg, immIOffset1 off)
 5733 %{
 5734   constraint(ALLOC_IN_RC(ptr_reg));
 5735   match(AddP reg off);
 5736   op_cost(0);
 5737   format %{ "[$reg, $off]" %}
 5738   interface(MEMORY_INTER) %{
 5739     base($reg);
 5740     index(0xffffffff);
 5741     scale(0x0);
 5742     disp($off);
 5743   %}
 5744 %}
 5745 
 5746 operand indOffI2(iRegP reg, immIOffset2 off)
 5747 %{
 5748   constraint(ALLOC_IN_RC(ptr_reg));
 5749   match(AddP reg off);
 5750   op_cost(0);
 5751   format %{ "[$reg, $off]" %}
 5752   interface(MEMORY_INTER) %{
 5753     base($reg);
 5754     index(0xffffffff);
 5755     scale(0x0);
 5756     disp($off);
 5757   %}
 5758 %}
 5759 
 5760 operand indOffI4(iRegP reg, immIOffset4 off)
 5761 %{
 5762   constraint(ALLOC_IN_RC(ptr_reg));
 5763   match(AddP reg off);
 5764   op_cost(0);
 5765   format %{ "[$reg, $off]" %}
 5766   interface(MEMORY_INTER) %{
 5767     base($reg);
 5768     index(0xffffffff);
 5769     scale(0x0);
 5770     disp($off);
 5771   %}
 5772 %}
 5773 
 5774 operand indOffI8(iRegP reg, immIOffset8 off)
 5775 %{
 5776   constraint(ALLOC_IN_RC(ptr_reg));
 5777   match(AddP reg off);
 5778   op_cost(0);
 5779   format %{ "[$reg, $off]" %}
 5780   interface(MEMORY_INTER) %{
 5781     base($reg);
 5782     index(0xffffffff);
 5783     scale(0x0);
 5784     disp($off);
 5785   %}
 5786 %}
 5787 
 5788 operand indOffI16(iRegP reg, immIOffset16 off)
 5789 %{
 5790   constraint(ALLOC_IN_RC(ptr_reg));
 5791   match(AddP reg off);
 5792   op_cost(0);
 5793   format %{ "[$reg, $off]" %}
 5794   interface(MEMORY_INTER) %{
 5795     base($reg);
 5796     index(0xffffffff);
 5797     scale(0x0);
 5798     disp($off);
 5799   %}
 5800 %}
 5801 
 5802 operand indOffL(iRegP reg, immLoffset off)
 5803 %{
 5804   constraint(ALLOC_IN_RC(ptr_reg));
 5805   match(AddP reg off);
 5806   op_cost(0);
 5807   format %{ "[$reg, $off]" %}
 5808   interface(MEMORY_INTER) %{
 5809     base($reg);
 5810     index(0xffffffff);
 5811     scale(0x0);
 5812     disp($off);
 5813   %}
 5814 %}
 5815 
 5816 operand indOffL1(iRegP reg, immLoffset1 off)
 5817 %{
 5818   constraint(ALLOC_IN_RC(ptr_reg));
 5819   match(AddP reg off);
 5820   op_cost(0);
 5821   format %{ "[$reg, $off]" %}
 5822   interface(MEMORY_INTER) %{
 5823     base($reg);
 5824     index(0xffffffff);
 5825     scale(0x0);
 5826     disp($off);
 5827   %}
 5828 %}
 5829 
 5830 operand indOffL2(iRegP reg, immLoffset2 off)
 5831 %{
 5832   constraint(ALLOC_IN_RC(ptr_reg));
 5833   match(AddP reg off);
 5834   op_cost(0);
 5835   format %{ "[$reg, $off]" %}
 5836   interface(MEMORY_INTER) %{
 5837     base($reg);
 5838     index(0xffffffff);
 5839     scale(0x0);
 5840     disp($off);
 5841   %}
 5842 %}
 5843 
 5844 operand indOffL4(iRegP reg, immLoffset4 off)
 5845 %{
 5846   constraint(ALLOC_IN_RC(ptr_reg));
 5847   match(AddP reg off);
 5848   op_cost(0);
 5849   format %{ "[$reg, $off]" %}
 5850   interface(MEMORY_INTER) %{
 5851     base($reg);
 5852     index(0xffffffff);
 5853     scale(0x0);
 5854     disp($off);
 5855   %}
 5856 %}
 5857 
 5858 operand indOffL8(iRegP reg, immLoffset8 off)
 5859 %{
 5860   constraint(ALLOC_IN_RC(ptr_reg));
 5861   match(AddP reg off);
 5862   op_cost(0);
 5863   format %{ "[$reg, $off]" %}
 5864   interface(MEMORY_INTER) %{
 5865     base($reg);
 5866     index(0xffffffff);
 5867     scale(0x0);
 5868     disp($off);
 5869   %}
 5870 %}
 5871 
 5872 operand indOffL16(iRegP reg, immLoffset16 off)
 5873 %{
 5874   constraint(ALLOC_IN_RC(ptr_reg));
 5875   match(AddP reg off);
 5876   op_cost(0);
 5877   format %{ "[$reg, $off]" %}
 5878   interface(MEMORY_INTER) %{
 5879     base($reg);
 5880     index(0xffffffff);
 5881     scale(0x0);
 5882     disp($off);
 5883   %}
 5884 %}
 5885 
 5886 operand indirectN(iRegN reg)
 5887 %{
 5888   predicate(CompressedOops::shift() == 0);
 5889   constraint(ALLOC_IN_RC(ptr_reg));
 5890   match(DecodeN reg);
 5891   op_cost(0);
 5892   format %{ "[$reg]\t# narrow" %}
 5893   interface(MEMORY_INTER) %{
 5894     base($reg);
 5895     index(0xffffffff);
 5896     scale(0x0);
 5897     disp(0x0);
 5898   %}
 5899 %}
 5900 
 5901 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5902 %{
 5903   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5904   constraint(ALLOC_IN_RC(ptr_reg));
 5905   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5906   op_cost(0);
 5907   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5908   interface(MEMORY_INTER) %{
 5909     base($reg);
 5910     index($ireg);
 5911     scale($scale);
 5912     disp(0x0);
 5913   %}
 5914 %}
 5915 
 5916 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5917 %{
 5918   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5919   constraint(ALLOC_IN_RC(ptr_reg));
 5920   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5921   op_cost(0);
 5922   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5923   interface(MEMORY_INTER) %{
 5924     base($reg);
 5925     index($lreg);
 5926     scale($scale);
 5927     disp(0x0);
 5928   %}
 5929 %}
 5930 
 5931 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5932 %{
 5933   predicate(CompressedOops::shift() == 0);
 5934   constraint(ALLOC_IN_RC(ptr_reg));
 5935   match(AddP (DecodeN reg) (ConvI2L ireg));
 5936   op_cost(0);
 5937   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5938   interface(MEMORY_INTER) %{
 5939     base($reg);
 5940     index($ireg);
 5941     scale(0x0);
 5942     disp(0x0);
 5943   %}
 5944 %}
 5945 
 5946 operand indIndexN(iRegN reg, iRegL lreg)
 5947 %{
 5948   predicate(CompressedOops::shift() == 0);
 5949   constraint(ALLOC_IN_RC(ptr_reg));
 5950   match(AddP (DecodeN reg) lreg);
 5951   op_cost(0);
 5952   format %{ "$reg, $lreg\t# narrow" %}
 5953   interface(MEMORY_INTER) %{
 5954     base($reg);
 5955     index($lreg);
 5956     scale(0x0);
 5957     disp(0x0);
 5958   %}
 5959 %}
 5960 
 5961 operand indOffIN(iRegN reg, immIOffset off)
 5962 %{
 5963   predicate(CompressedOops::shift() == 0);
 5964   constraint(ALLOC_IN_RC(ptr_reg));
 5965   match(AddP (DecodeN reg) off);
 5966   op_cost(0);
 5967   format %{ "[$reg, $off]\t# narrow" %}
 5968   interface(MEMORY_INTER) %{
 5969     base($reg);
 5970     index(0xffffffff);
 5971     scale(0x0);
 5972     disp($off);
 5973   %}
 5974 %}
 5975 
 5976 operand indOffLN(iRegN reg, immLoffset off)
 5977 %{
 5978   predicate(CompressedOops::shift() == 0);
 5979   constraint(ALLOC_IN_RC(ptr_reg));
 5980   match(AddP (DecodeN reg) off);
 5981   op_cost(0);
 5982   format %{ "[$reg, $off]\t# narrow" %}
 5983   interface(MEMORY_INTER) %{
 5984     base($reg);
 5985     index(0xffffffff);
 5986     scale(0x0);
 5987     disp($off);
 5988   %}
 5989 %}
 5990 
 5991 
 5992 
 5993 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5994 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5995 %{
 5996   constraint(ALLOC_IN_RC(ptr_reg));
 5997   match(AddP reg off);
 5998   op_cost(0);
 5999   format %{ "[$reg, $off]" %}
 6000   interface(MEMORY_INTER) %{
 6001     base($reg);
 6002     index(0xffffffff);
 6003     scale(0x0);
 6004     disp($off);
 6005   %}
 6006 %}
 6007 
 6008 //----------Special Memory Operands--------------------------------------------
 6009 // Stack Slot Operand - This operand is used for loading and storing temporary
 6010 //                      values on the stack where a match requires a value to
 6011 //                      flow through memory.
 6012 operand stackSlotP(sRegP reg)
 6013 %{
 6014   constraint(ALLOC_IN_RC(stack_slots));
 6015   op_cost(100);
 6016   // No match rule because this operand is only generated in matching
 6017   // match(RegP);
 6018   format %{ "[$reg]" %}
 6019   interface(MEMORY_INTER) %{
 6020     base(0x1e);  // RSP
 6021     index(0x0);  // No Index
 6022     scale(0x0);  // No Scale
 6023     disp($reg);  // Stack Offset
 6024   %}
 6025 %}
 6026 
 6027 operand stackSlotI(sRegI reg)
 6028 %{
 6029   constraint(ALLOC_IN_RC(stack_slots));
 6030   // No match rule because this operand is only generated in matching
 6031   // match(RegI);
 6032   format %{ "[$reg]" %}
 6033   interface(MEMORY_INTER) %{
 6034     base(0x1e);  // RSP
 6035     index(0x0);  // No Index
 6036     scale(0x0);  // No Scale
 6037     disp($reg);  // Stack Offset
 6038   %}
 6039 %}
 6040 
 6041 operand stackSlotF(sRegF reg)
 6042 %{
 6043   constraint(ALLOC_IN_RC(stack_slots));
 6044   // No match rule because this operand is only generated in matching
 6045   // match(RegF);
 6046   format %{ "[$reg]" %}
 6047   interface(MEMORY_INTER) %{
 6048     base(0x1e);  // RSP
 6049     index(0x0);  // No Index
 6050     scale(0x0);  // No Scale
 6051     disp($reg);  // Stack Offset
 6052   %}
 6053 %}
 6054 
 6055 operand stackSlotD(sRegD reg)
 6056 %{
 6057   constraint(ALLOC_IN_RC(stack_slots));
 6058   // No match rule because this operand is only generated in matching
 6059   // match(RegD);
 6060   format %{ "[$reg]" %}
 6061   interface(MEMORY_INTER) %{
 6062     base(0x1e);  // RSP
 6063     index(0x0);  // No Index
 6064     scale(0x0);  // No Scale
 6065     disp($reg);  // Stack Offset
 6066   %}
 6067 %}
 6068 
 6069 operand stackSlotL(sRegL reg)
 6070 %{
 6071   constraint(ALLOC_IN_RC(stack_slots));
 6072   // No match rule because this operand is only generated in matching
 6073   // match(RegL);
 6074   format %{ "[$reg]" %}
 6075   interface(MEMORY_INTER) %{
 6076     base(0x1e);  // RSP
 6077     index(0x0);  // No Index
 6078     scale(0x0);  // No Scale
 6079     disp($reg);  // Stack Offset
 6080   %}
 6081 %}
 6082 
 6083 // Operands for expressing Control Flow
 6084 // NOTE: Label is a predefined operand which should not be redefined in
 6085 //       the AD file. It is generically handled within the ADLC.
 6086 
 6087 //----------Conditional Branch Operands----------------------------------------
 6088 // Comparison Op  - This is the operation of the comparison, and is limited to
 6089 //                  the following set of codes:
 6090 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6091 //
 6092 // Other attributes of the comparison, such as unsignedness, are specified
 6093 // by the comparison instruction that sets a condition code flags register.
 6094 // That result is represented by a flags operand whose subtype is appropriate
 6095 // to the unsignedness (etc.) of the comparison.
 6096 //
 6097 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6098 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6099 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6100 
 6101 // used for signed integral comparisons and fp comparisons
 6102 
 6103 operand cmpOp()
 6104 %{
 6105   match(Bool);
 6106 
 6107   format %{ "" %}
 6108   interface(COND_INTER) %{
 6109     equal(0x0, "eq");
 6110     not_equal(0x1, "ne");
 6111     less(0xb, "lt");
 6112     greater_equal(0xa, "ge");
 6113     less_equal(0xd, "le");
 6114     greater(0xc, "gt");
 6115     overflow(0x6, "vs");
 6116     no_overflow(0x7, "vc");
 6117   %}
 6118 %}
 6119 
 6120 // used for unsigned integral comparisons
 6121 
 6122 operand cmpOpU()
 6123 %{
 6124   match(Bool);
 6125 
 6126   format %{ "" %}
 6127   interface(COND_INTER) %{
 6128     equal(0x0, "eq");
 6129     not_equal(0x1, "ne");
 6130     less(0x3, "lo");
 6131     greater_equal(0x2, "hs");
 6132     less_equal(0x9, "ls");
 6133     greater(0x8, "hi");
 6134     overflow(0x6, "vs");
 6135     no_overflow(0x7, "vc");
 6136   %}
 6137 %}
 6138 
 6139 // used for certain integral comparisons which can be
 6140 // converted to cbxx or tbxx instructions
 6141 
 6142 operand cmpOpEqNe()
 6143 %{
 6144   match(Bool);
 6145   op_cost(0);
 6146   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6147             || n->as_Bool()->_test._test == BoolTest::eq);
 6148 
 6149   format %{ "" %}
 6150   interface(COND_INTER) %{
 6151     equal(0x0, "eq");
 6152     not_equal(0x1, "ne");
 6153     less(0xb, "lt");
 6154     greater_equal(0xa, "ge");
 6155     less_equal(0xd, "le");
 6156     greater(0xc, "gt");
 6157     overflow(0x6, "vs");
 6158     no_overflow(0x7, "vc");
 6159   %}
 6160 %}
 6161 
 6162 // used for certain integral comparisons which can be
 6163 // converted to cbxx or tbxx instructions
 6164 
 6165 operand cmpOpLtGe()
 6166 %{
 6167   match(Bool);
 6168   op_cost(0);
 6169 
 6170   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6171             || n->as_Bool()->_test._test == BoolTest::ge);
 6172 
 6173   format %{ "" %}
 6174   interface(COND_INTER) %{
 6175     equal(0x0, "eq");
 6176     not_equal(0x1, "ne");
 6177     less(0xb, "lt");
 6178     greater_equal(0xa, "ge");
 6179     less_equal(0xd, "le");
 6180     greater(0xc, "gt");
 6181     overflow(0x6, "vs");
 6182     no_overflow(0x7, "vc");
 6183   %}
 6184 %}
 6185 
 6186 // used for certain unsigned integral comparisons which can be
 6187 // converted to cbxx or tbxx instructions
 6188 
 6189 operand cmpOpUEqNeLtGe()
 6190 %{
 6191   match(Bool);
 6192   op_cost(0);
 6193 
 6194   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6195             || n->as_Bool()->_test._test == BoolTest::ne
 6196             || n->as_Bool()->_test._test == BoolTest::lt
 6197             || n->as_Bool()->_test._test == BoolTest::ge);
 6198 
 6199   format %{ "" %}
 6200   interface(COND_INTER) %{
 6201     equal(0x0, "eq");
 6202     not_equal(0x1, "ne");
 6203     less(0xb, "lt");
 6204     greater_equal(0xa, "ge");
 6205     less_equal(0xd, "le");
 6206     greater(0xc, "gt");
 6207     overflow(0x6, "vs");
 6208     no_overflow(0x7, "vc");
 6209   %}
 6210 %}
 6211 
 6212 // Special operand allowing long args to int ops to be truncated for free
 6213 
 6214 operand iRegL2I(iRegL reg) %{
 6215 
 6216   op_cost(0);
 6217 
 6218   match(ConvL2I reg);
 6219 
 6220   format %{ "l2i($reg)" %}
 6221 
 6222   interface(REG_INTER)
 6223 %}
 6224 
 6225 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6226 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6227 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6228 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6229 
 6230 //----------OPERAND CLASSES----------------------------------------------------
 6231 // Operand Classes are groups of operands that are used as to simplify
 6232 // instruction definitions by not requiring the AD writer to specify
 6233 // separate instructions for every form of operand when the
 6234 // instruction accepts multiple operand types with the same basic
 6235 // encoding and format. The classic case of this is memory operands.
 6236 
 6237 // memory is used to define read/write location for load/store
 6238 // instruction defs. we can turn a memory op into an Address
 6239 
 6240 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6241                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6242 
 6243 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6244                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6245 
 6246 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6247                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6248 
 6249 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6250                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6251 
 6252 // All of the memory operands. For the pipeline description.
 6253 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6254                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6255                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6256 
 6257 
 6258 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6259 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6260 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6261 // can be elided because the 32-bit instruction will just employ the
 6262 // lower 32 bits anyway.
 6263 //
 6264 // n.b. this does not elide all L2I conversions. if the truncated
 6265 // value is consumed by more than one operation then the ConvL2I
 6266 // cannot be bundled into the consuming nodes so an l2i gets planted
 6267 // (actually a movw $dst $src) and the downstream instructions consume
 6268 // the result of the l2i as an iRegI input. That's a shame since the
 6269 // movw is actually redundant but its not too costly.
 6270 
 6271 opclass iRegIorL2I(iRegI, iRegL2I);
 6272 
 6273 //----------PIPELINE-----------------------------------------------------------
 6274 // Rules which define the behavior of the target architectures pipeline.
 6275 
 6276 // For specific pipelines, eg A53, define the stages of that pipeline
 6277 //pipe_desc(ISS, EX1, EX2, WR);
 6278 #define ISS S0
 6279 #define EX1 S1
 6280 #define EX2 S2
 6281 #define WR  S3
 6282 
 6283 // Integer ALU reg operation
 6284 pipeline %{
 6285 
 6286 attributes %{
 6287   // ARM instructions are of fixed length
 6288   fixed_size_instructions;        // Fixed size instructions TODO does
 6289   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6290   // ARM instructions come in 32-bit word units
 6291   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6292   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6293   instruction_fetch_units = 1;       // of 64 bytes
 6294 
 6295   // List of nop instructions
 6296   nops( MachNop );
 6297 %}
 6298 
 6299 // We don't use an actual pipeline model so don't care about resources
 6300 // or description. we do use pipeline classes to introduce fixed
 6301 // latencies
 6302 
 6303 //----------RESOURCES----------------------------------------------------------
 6304 // Resources are the functional units available to the machine
 6305 
 6306 resources( INS0, INS1, INS01 = INS0 | INS1,
 6307            ALU0, ALU1, ALU = ALU0 | ALU1,
 6308            MAC,
 6309            DIV,
 6310            BRANCH,
 6311            LDST,
 6312            NEON_FP);
 6313 
 6314 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6315 // Pipeline Description specifies the stages in the machine's pipeline
 6316 
 6317 // Define the pipeline as a generic 6 stage pipeline
 6318 pipe_desc(S0, S1, S2, S3, S4, S5);
 6319 
 6320 //----------PIPELINE CLASSES---------------------------------------------------
 6321 // Pipeline Classes describe the stages in which input and output are
 6322 // referenced by the hardware pipeline.
 6323 
 6324 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6325 %{
 6326   single_instruction;
 6327   src1   : S1(read);
 6328   src2   : S2(read);
 6329   dst    : S5(write);
 6330   INS01  : ISS;
 6331   NEON_FP : S5;
 6332 %}
 6333 
 6334 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6335 %{
 6336   single_instruction;
 6337   src1   : S1(read);
 6338   src2   : S2(read);
 6339   dst    : S5(write);
 6340   INS01  : ISS;
 6341   NEON_FP : S5;
 6342 %}
 6343 
 6344 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6345 %{
 6346   single_instruction;
 6347   src    : S1(read);
 6348   dst    : S5(write);
 6349   INS01  : ISS;
 6350   NEON_FP : S5;
 6351 %}
 6352 
 6353 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6354 %{
 6355   single_instruction;
 6356   src    : S1(read);
 6357   dst    : S5(write);
 6358   INS01  : ISS;
 6359   NEON_FP : S5;
 6360 %}
 6361 
 6362 pipe_class fp_d2f(vRegF dst, vRegD src)
 6363 %{
 6364   single_instruction;
 6365   src    : S1(read);
 6366   dst    : S5(write);
 6367   INS01  : ISS;
 6368   NEON_FP : S5;
 6369 %}
 6370 
 6371 pipe_class fp_f2d(vRegD dst, vRegF src)
 6372 %{
 6373   single_instruction;
 6374   src    : S1(read);
 6375   dst    : S5(write);
 6376   INS01  : ISS;
 6377   NEON_FP : S5;
 6378 %}
 6379 
 6380 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6381 %{
 6382   single_instruction;
 6383   src    : S1(read);
 6384   dst    : S5(write);
 6385   INS01  : ISS;
 6386   NEON_FP : S5;
 6387 %}
 6388 
 6389 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6390 %{
 6391   single_instruction;
 6392   src    : S1(read);
 6393   dst    : S5(write);
 6394   INS01  : ISS;
 6395   NEON_FP : S5;
 6396 %}
 6397 
 6398 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6399 %{
 6400   single_instruction;
 6401   src    : S1(read);
 6402   dst    : S5(write);
 6403   INS01  : ISS;
 6404   NEON_FP : S5;
 6405 %}
 6406 
 6407 pipe_class fp_l2f(vRegF dst, iRegL src)
 6408 %{
 6409   single_instruction;
 6410   src    : S1(read);
 6411   dst    : S5(write);
 6412   INS01  : ISS;
 6413   NEON_FP : S5;
 6414 %}
 6415 
 6416 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6417 %{
 6418   single_instruction;
 6419   src    : S1(read);
 6420   dst    : S5(write);
 6421   INS01  : ISS;
 6422   NEON_FP : S5;
 6423 %}
 6424 
 6425 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6426 %{
 6427   single_instruction;
 6428   src    : S1(read);
 6429   dst    : S5(write);
 6430   INS01  : ISS;
 6431   NEON_FP : S5;
 6432 %}
 6433 
 6434 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6435 %{
 6436   single_instruction;
 6437   src    : S1(read);
 6438   dst    : S5(write);
 6439   INS01  : ISS;
 6440   NEON_FP : S5;
 6441 %}
 6442 
 6443 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6444 %{
 6445   single_instruction;
 6446   src    : S1(read);
 6447   dst    : S5(write);
 6448   INS01  : ISS;
 6449   NEON_FP : S5;
 6450 %}
 6451 
 6452 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6453 %{
 6454   single_instruction;
 6455   src1   : S1(read);
 6456   src2   : S2(read);
 6457   dst    : S5(write);
 6458   INS0   : ISS;
 6459   NEON_FP : S5;
 6460 %}
 6461 
 6462 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6463 %{
 6464   single_instruction;
 6465   src1   : S1(read);
 6466   src2   : S2(read);
 6467   dst    : S5(write);
 6468   INS0   : ISS;
 6469   NEON_FP : S5;
 6470 %}
 6471 
 6472 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6473 %{
 6474   single_instruction;
 6475   cr     : S1(read);
 6476   src1   : S1(read);
 6477   src2   : S1(read);
 6478   dst    : S3(write);
 6479   INS01  : ISS;
 6480   NEON_FP : S3;
 6481 %}
 6482 
 6483 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6484 %{
 6485   single_instruction;
 6486   cr     : S1(read);
 6487   src1   : S1(read);
 6488   src2   : S1(read);
 6489   dst    : S3(write);
 6490   INS01  : ISS;
 6491   NEON_FP : S3;
 6492 %}
 6493 
 6494 pipe_class fp_imm_s(vRegF dst)
 6495 %{
 6496   single_instruction;
 6497   dst    : S3(write);
 6498   INS01  : ISS;
 6499   NEON_FP : S3;
 6500 %}
 6501 
 6502 pipe_class fp_imm_d(vRegD dst)
 6503 %{
 6504   single_instruction;
 6505   dst    : S3(write);
 6506   INS01  : ISS;
 6507   NEON_FP : S3;
 6508 %}
 6509 
 6510 pipe_class fp_load_constant_s(vRegF dst)
 6511 %{
 6512   single_instruction;
 6513   dst    : S4(write);
 6514   INS01  : ISS;
 6515   NEON_FP : S4;
 6516 %}
 6517 
 6518 pipe_class fp_load_constant_d(vRegD dst)
 6519 %{
 6520   single_instruction;
 6521   dst    : S4(write);
 6522   INS01  : ISS;
 6523   NEON_FP : S4;
 6524 %}
 6525 
 6526 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6527 %{
 6528   single_instruction;
 6529   dst    : S5(write);
 6530   src1   : S1(read);
 6531   src2   : S1(read);
 6532   INS01  : ISS;
 6533   NEON_FP : S5;
 6534 %}
 6535 
 6536 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6537 %{
 6538   single_instruction;
 6539   dst    : S5(write);
 6540   src1   : S1(read);
 6541   src2   : S1(read);
 6542   INS0   : ISS;
 6543   NEON_FP : S5;
 6544 %}
 6545 
 6546 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6547 %{
 6548   single_instruction;
 6549   dst    : S5(write);
 6550   src1   : S1(read);
 6551   src2   : S1(read);
 6552   dst    : S1(read);
 6553   INS01  : ISS;
 6554   NEON_FP : S5;
 6555 %}
 6556 
 6557 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6558 %{
 6559   single_instruction;
 6560   dst    : S5(write);
 6561   src1   : S1(read);
 6562   src2   : S1(read);
 6563   dst    : S1(read);
 6564   INS0   : ISS;
 6565   NEON_FP : S5;
 6566 %}
 6567 
 6568 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6569 %{
 6570   single_instruction;
 6571   dst    : S4(write);
 6572   src1   : S2(read);
 6573   src2   : S2(read);
 6574   INS01  : ISS;
 6575   NEON_FP : S4;
 6576 %}
 6577 
 6578 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6579 %{
 6580   single_instruction;
 6581   dst    : S4(write);
 6582   src1   : S2(read);
 6583   src2   : S2(read);
 6584   INS0   : ISS;
 6585   NEON_FP : S4;
 6586 %}
 6587 
 6588 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6589 %{
 6590   single_instruction;
 6591   dst    : S3(write);
 6592   src1   : S2(read);
 6593   src2   : S2(read);
 6594   INS01  : ISS;
 6595   NEON_FP : S3;
 6596 %}
 6597 
 6598 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6599 %{
 6600   single_instruction;
 6601   dst    : S3(write);
 6602   src1   : S2(read);
 6603   src2   : S2(read);
 6604   INS0   : ISS;
 6605   NEON_FP : S3;
 6606 %}
 6607 
 6608 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6609 %{
 6610   single_instruction;
 6611   dst    : S3(write);
 6612   src    : S1(read);
 6613   shift  : S1(read);
 6614   INS01  : ISS;
 6615   NEON_FP : S3;
 6616 %}
 6617 
 6618 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6619 %{
 6620   single_instruction;
 6621   dst    : S3(write);
 6622   src    : S1(read);
 6623   shift  : S1(read);
 6624   INS0   : ISS;
 6625   NEON_FP : S3;
 6626 %}
 6627 
 6628 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6629 %{
 6630   single_instruction;
 6631   dst    : S3(write);
 6632   src    : S1(read);
 6633   INS01  : ISS;
 6634   NEON_FP : S3;
 6635 %}
 6636 
 6637 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6638 %{
 6639   single_instruction;
 6640   dst    : S3(write);
 6641   src    : S1(read);
 6642   INS0   : ISS;
 6643   NEON_FP : S3;
 6644 %}
 6645 
 6646 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6647 %{
 6648   single_instruction;
 6649   dst    : S5(write);
 6650   src1   : S1(read);
 6651   src2   : S1(read);
 6652   INS01  : ISS;
 6653   NEON_FP : S5;
 6654 %}
 6655 
 6656 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6657 %{
 6658   single_instruction;
 6659   dst    : S5(write);
 6660   src1   : S1(read);
 6661   src2   : S1(read);
 6662   INS0   : ISS;
 6663   NEON_FP : S5;
 6664 %}
 6665 
 6666 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6667 %{
 6668   single_instruction;
 6669   dst    : S5(write);
 6670   src1   : S1(read);
 6671   src2   : S1(read);
 6672   INS0   : ISS;
 6673   NEON_FP : S5;
 6674 %}
 6675 
 6676 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6677 %{
 6678   single_instruction;
 6679   dst    : S5(write);
 6680   src1   : S1(read);
 6681   src2   : S1(read);
 6682   INS0   : ISS;
 6683   NEON_FP : S5;
 6684 %}
 6685 
 6686 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6687 %{
 6688   single_instruction;
 6689   dst    : S5(write);
 6690   src    : S1(read);
 6691   INS0   : ISS;
 6692   NEON_FP : S5;
 6693 %}
 6694 
 6695 pipe_class vunop_fp64(vecD dst, vecD src)
 6696 %{
 6697   single_instruction;
 6698   dst    : S5(write);
 6699   src    : S1(read);
 6700   INS01  : ISS;
 6701   NEON_FP : S5;
 6702 %}
 6703 
 6704 pipe_class vunop_fp128(vecX dst, vecX src)
 6705 %{
 6706   single_instruction;
 6707   dst    : S5(write);
 6708   src    : S1(read);
 6709   INS0   : ISS;
 6710   NEON_FP : S5;
 6711 %}
 6712 
 6713 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6714 %{
 6715   single_instruction;
 6716   dst    : S3(write);
 6717   src    : S1(read);
 6718   INS01  : ISS;
 6719   NEON_FP : S3;
 6720 %}
 6721 
 6722 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6723 %{
 6724   single_instruction;
 6725   dst    : S3(write);
 6726   src    : S1(read);
 6727   INS01  : ISS;
 6728   NEON_FP : S3;
 6729 %}
 6730 
 6731 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6732 %{
 6733   single_instruction;
 6734   dst    : S3(write);
 6735   src    : S1(read);
 6736   INS01  : ISS;
 6737   NEON_FP : S3;
 6738 %}
 6739 
 6740 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6741 %{
 6742   single_instruction;
 6743   dst    : S3(write);
 6744   src    : S1(read);
 6745   INS01  : ISS;
 6746   NEON_FP : S3;
 6747 %}
 6748 
 6749 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6750 %{
 6751   single_instruction;
 6752   dst    : S3(write);
 6753   src    : S1(read);
 6754   INS01  : ISS;
 6755   NEON_FP : S3;
 6756 %}
 6757 
 6758 pipe_class vmovi_reg_imm64(vecD dst)
 6759 %{
 6760   single_instruction;
 6761   dst    : S3(write);
 6762   INS01  : ISS;
 6763   NEON_FP : S3;
 6764 %}
 6765 
 6766 pipe_class vmovi_reg_imm128(vecX dst)
 6767 %{
 6768   single_instruction;
 6769   dst    : S3(write);
 6770   INS0   : ISS;
 6771   NEON_FP : S3;
 6772 %}
 6773 
 6774 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6775 %{
 6776   single_instruction;
 6777   dst    : S5(write);
 6778   mem    : ISS(read);
 6779   INS01  : ISS;
 6780   NEON_FP : S3;
 6781 %}
 6782 
 6783 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6784 %{
 6785   single_instruction;
 6786   dst    : S5(write);
 6787   mem    : ISS(read);
 6788   INS01  : ISS;
 6789   NEON_FP : S3;
 6790 %}
 6791 
 6792 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6793 %{
 6794   single_instruction;
 6795   mem    : ISS(read);
 6796   src    : S2(read);
 6797   INS01  : ISS;
 6798   NEON_FP : S3;
 6799 %}
 6800 
 6801 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6802 %{
 6803   single_instruction;
 6804   mem    : ISS(read);
 6805   src    : S2(read);
 6806   INS01  : ISS;
 6807   NEON_FP : S3;
 6808 %}
 6809 
 6810 //------- Integer ALU operations --------------------------
 6811 
 6812 // Integer ALU reg-reg operation
 6813 // Operands needed in EX1, result generated in EX2
 6814 // Eg.  ADD     x0, x1, x2
 6815 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6816 %{
 6817   single_instruction;
 6818   dst    : EX2(write);
 6819   src1   : EX1(read);
 6820   src2   : EX1(read);
 6821   INS01  : ISS; // Dual issue as instruction 0 or 1
 6822   ALU    : EX2;
 6823 %}
 6824 
 6825 // Integer ALU reg-reg operation with constant shift
 6826 // Shifted register must be available in LATE_ISS instead of EX1
 6827 // Eg.  ADD     x0, x1, x2, LSL #2
 6828 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6829 %{
 6830   single_instruction;
 6831   dst    : EX2(write);
 6832   src1   : EX1(read);
 6833   src2   : ISS(read);
 6834   INS01  : ISS;
 6835   ALU    : EX2;
 6836 %}
 6837 
 6838 // Integer ALU reg operation with constant shift
 6839 // Eg.  LSL     x0, x1, #shift
 6840 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6841 %{
 6842   single_instruction;
 6843   dst    : EX2(write);
 6844   src1   : ISS(read);
 6845   INS01  : ISS;
 6846   ALU    : EX2;
 6847 %}
 6848 
 6849 // Integer ALU reg-reg operation with variable shift
 6850 // Both operands must be available in LATE_ISS instead of EX1
 6851 // Result is available in EX1 instead of EX2
 6852 // Eg.  LSLV    x0, x1, x2
 6853 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6854 %{
 6855   single_instruction;
 6856   dst    : EX1(write);
 6857   src1   : ISS(read);
 6858   src2   : ISS(read);
 6859   INS01  : ISS;
 6860   ALU    : EX1;
 6861 %}
 6862 
 6863 // Integer ALU reg-reg operation with extract
 6864 // As for _vshift above, but result generated in EX2
 6865 // Eg.  EXTR    x0, x1, x2, #N
 6866 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6867 %{
 6868   single_instruction;
 6869   dst    : EX2(write);
 6870   src1   : ISS(read);
 6871   src2   : ISS(read);
 6872   INS1   : ISS; // Can only dual issue as Instruction 1
 6873   ALU    : EX1;
 6874 %}
 6875 
 6876 // Integer ALU reg operation
 6877 // Eg.  NEG     x0, x1
 6878 pipe_class ialu_reg(iRegI dst, iRegI src)
 6879 %{
 6880   single_instruction;
 6881   dst    : EX2(write);
 6882   src    : EX1(read);
 6883   INS01  : ISS;
 6884   ALU    : EX2;
 6885 %}
 6886 
 6887 // Integer ALU reg mmediate operation
 6888 // Eg.  ADD     x0, x1, #N
 6889 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6890 %{
 6891   single_instruction;
 6892   dst    : EX2(write);
 6893   src1   : EX1(read);
 6894   INS01  : ISS;
 6895   ALU    : EX2;
 6896 %}
 6897 
 6898 // Integer ALU immediate operation (no source operands)
 6899 // Eg.  MOV     x0, #N
 6900 pipe_class ialu_imm(iRegI dst)
 6901 %{
 6902   single_instruction;
 6903   dst    : EX1(write);
 6904   INS01  : ISS;
 6905   ALU    : EX1;
 6906 %}
 6907 
 6908 //------- Compare operation -------------------------------
 6909 
 6910 // Compare reg-reg
 6911 // Eg.  CMP     x0, x1
 6912 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6913 %{
 6914   single_instruction;
 6915 //  fixed_latency(16);
 6916   cr     : EX2(write);
 6917   op1    : EX1(read);
 6918   op2    : EX1(read);
 6919   INS01  : ISS;
 6920   ALU    : EX2;
 6921 %}
 6922 
 6923 // Compare reg-reg
 6924 // Eg.  CMP     x0, #N
 6925 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6926 %{
 6927   single_instruction;
 6928 //  fixed_latency(16);
 6929   cr     : EX2(write);
 6930   op1    : EX1(read);
 6931   INS01  : ISS;
 6932   ALU    : EX2;
 6933 %}
 6934 
 6935 //------- Conditional instructions ------------------------
 6936 
 6937 // Conditional no operands
 6938 // Eg.  CSINC   x0, zr, zr, <cond>
 6939 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6940 %{
 6941   single_instruction;
 6942   cr     : EX1(read);
 6943   dst    : EX2(write);
 6944   INS01  : ISS;
 6945   ALU    : EX2;
 6946 %}
 6947 
 6948 // Conditional 2 operand
 6949 // EG.  CSEL    X0, X1, X2, <cond>
 6950 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6951 %{
 6952   single_instruction;
 6953   cr     : EX1(read);
 6954   src1   : EX1(read);
 6955   src2   : EX1(read);
 6956   dst    : EX2(write);
 6957   INS01  : ISS;
 6958   ALU    : EX2;
 6959 %}
 6960 
 6961 // Conditional 2 operand
 6962 // EG.  CSEL    X0, X1, X2, <cond>
 6963 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6964 %{
 6965   single_instruction;
 6966   cr     : EX1(read);
 6967   src    : EX1(read);
 6968   dst    : EX2(write);
 6969   INS01  : ISS;
 6970   ALU    : EX2;
 6971 %}
 6972 
 6973 //------- Multiply pipeline operations --------------------
 6974 
 6975 // Multiply reg-reg
 6976 // Eg.  MUL     w0, w1, w2
 6977 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6978 %{
 6979   single_instruction;
 6980   dst    : WR(write);
 6981   src1   : ISS(read);
 6982   src2   : ISS(read);
 6983   INS01  : ISS;
 6984   MAC    : WR;
 6985 %}
 6986 
 6987 // Multiply accumulate
 6988 // Eg.  MADD    w0, w1, w2, w3
 6989 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6990 %{
 6991   single_instruction;
 6992   dst    : WR(write);
 6993   src1   : ISS(read);
 6994   src2   : ISS(read);
 6995   src3   : ISS(read);
 6996   INS01  : ISS;
 6997   MAC    : WR;
 6998 %}
 6999 
 7000 // Eg.  MUL     w0, w1, w2
 7001 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7002 %{
 7003   single_instruction;
 7004   fixed_latency(3); // Maximum latency for 64 bit mul
 7005   dst    : WR(write);
 7006   src1   : ISS(read);
 7007   src2   : ISS(read);
 7008   INS01  : ISS;
 7009   MAC    : WR;
 7010 %}
 7011 
 7012 // Multiply accumulate
 7013 // Eg.  MADD    w0, w1, w2, w3
 7014 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 7015 %{
 7016   single_instruction;
 7017   fixed_latency(3); // Maximum latency for 64 bit mul
 7018   dst    : WR(write);
 7019   src1   : ISS(read);
 7020   src2   : ISS(read);
 7021   src3   : ISS(read);
 7022   INS01  : ISS;
 7023   MAC    : WR;
 7024 %}
 7025 
 7026 //------- Divide pipeline operations --------------------
 7027 
 7028 // Eg.  SDIV    w0, w1, w2
 7029 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7030 %{
 7031   single_instruction;
 7032   fixed_latency(8); // Maximum latency for 32 bit divide
 7033   dst    : WR(write);
 7034   src1   : ISS(read);
 7035   src2   : ISS(read);
 7036   INS0   : ISS; // Can only dual issue as instruction 0
 7037   DIV    : WR;
 7038 %}
 7039 
 7040 // Eg.  SDIV    x0, x1, x2
 7041 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7042 %{
 7043   single_instruction;
 7044   fixed_latency(16); // Maximum latency for 64 bit divide
 7045   dst    : WR(write);
 7046   src1   : ISS(read);
 7047   src2   : ISS(read);
 7048   INS0   : ISS; // Can only dual issue as instruction 0
 7049   DIV    : WR;
 7050 %}
 7051 
 7052 //------- Load pipeline operations ------------------------
 7053 
 7054 // Load - prefetch
 7055 // Eg.  PFRM    <mem>
 7056 pipe_class iload_prefetch(memory mem)
 7057 %{
 7058   single_instruction;
 7059   mem    : ISS(read);
 7060   INS01  : ISS;
 7061   LDST   : WR;
 7062 %}
 7063 
 7064 // Load - reg, mem
 7065 // Eg.  LDR     x0, <mem>
 7066 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7067 %{
 7068   single_instruction;
 7069   dst    : WR(write);
 7070   mem    : ISS(read);
 7071   INS01  : ISS;
 7072   LDST   : WR;
 7073 %}
 7074 
 7075 // Load - reg, reg
 7076 // Eg.  LDR     x0, [sp, x1]
 7077 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7078 %{
 7079   single_instruction;
 7080   dst    : WR(write);
 7081   src    : ISS(read);
 7082   INS01  : ISS;
 7083   LDST   : WR;
 7084 %}
 7085 
 7086 //------- Store pipeline operations -----------------------
 7087 
 7088 // Store - zr, mem
 7089 // Eg.  STR     zr, <mem>
 7090 pipe_class istore_mem(memory mem)
 7091 %{
 7092   single_instruction;
 7093   mem    : ISS(read);
 7094   INS01  : ISS;
 7095   LDST   : WR;
 7096 %}
 7097 
 7098 // Store - reg, mem
 7099 // Eg.  STR     x0, <mem>
 7100 pipe_class istore_reg_mem(iRegI src, memory mem)
 7101 %{
 7102   single_instruction;
 7103   mem    : ISS(read);
 7104   src    : EX2(read);
 7105   INS01  : ISS;
 7106   LDST   : WR;
 7107 %}
 7108 
 7109 // Store - reg, reg
 7110 // Eg. STR      x0, [sp, x1]
 7111 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7112 %{
 7113   single_instruction;
 7114   dst    : ISS(read);
 7115   src    : EX2(read);
 7116   INS01  : ISS;
 7117   LDST   : WR;
 7118 %}
 7119 
 7120 //------- Store pipeline operations -----------------------
 7121 
 7122 // Branch
 7123 pipe_class pipe_branch()
 7124 %{
 7125   single_instruction;
 7126   INS01  : ISS;
 7127   BRANCH : EX1;
 7128 %}
 7129 
 7130 // Conditional branch
 7131 pipe_class pipe_branch_cond(rFlagsReg cr)
 7132 %{
 7133   single_instruction;
 7134   cr     : EX1(read);
 7135   INS01  : ISS;
 7136   BRANCH : EX1;
 7137 %}
 7138 
 7139 // Compare & Branch
 7140 // EG.  CBZ/CBNZ
 7141 pipe_class pipe_cmp_branch(iRegI op1)
 7142 %{
 7143   single_instruction;
 7144   op1    : EX1(read);
 7145   INS01  : ISS;
 7146   BRANCH : EX1;
 7147 %}
 7148 
 7149 //------- Synchronisation operations ----------------------
 7150 
 7151 // Any operation requiring serialization.
 7152 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7153 pipe_class pipe_serial()
 7154 %{
 7155   single_instruction;
 7156   force_serialization;
 7157   fixed_latency(16);
 7158   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7159   LDST   : WR;
 7160 %}
 7161 
 7162 // Generic big/slow expanded idiom - also serialized
 7163 pipe_class pipe_slow()
 7164 %{
 7165   instruction_count(10);
 7166   multiple_bundles;
 7167   force_serialization;
 7168   fixed_latency(16);
 7169   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7170   LDST   : WR;
 7171 %}
 7172 
 7173 // Empty pipeline class
 7174 pipe_class pipe_class_empty()
 7175 %{
 7176   single_instruction;
 7177   fixed_latency(0);
 7178 %}
 7179 
 7180 // Default pipeline class.
 7181 pipe_class pipe_class_default()
 7182 %{
 7183   single_instruction;
 7184   fixed_latency(2);
 7185 %}
 7186 
 7187 // Pipeline class for compares.
 7188 pipe_class pipe_class_compare()
 7189 %{
 7190   single_instruction;
 7191   fixed_latency(16);
 7192 %}
 7193 
 7194 // Pipeline class for memory operations.
 7195 pipe_class pipe_class_memory()
 7196 %{
 7197   single_instruction;
 7198   fixed_latency(16);
 7199 %}
 7200 
 7201 // Pipeline class for call.
 7202 pipe_class pipe_class_call()
 7203 %{
 7204   single_instruction;
 7205   fixed_latency(100);
 7206 %}
 7207 
 7208 // Define the class for the Nop node.
 7209 define %{
 7210    MachNop = pipe_class_empty;
 7211 %}
 7212 
 7213 %}
 7214 //----------INSTRUCTIONS-------------------------------------------------------
 7215 //
 7216 // match      -- States which machine-independent subtree may be replaced
 7217 //               by this instruction.
 7218 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7219 //               selection to identify a minimum cost tree of machine
 7220 //               instructions that matches a tree of machine-independent
 7221 //               instructions.
 7222 // format     -- A string providing the disassembly for this instruction.
 7223 //               The value of an instruction's operand may be inserted
 7224 //               by referring to it with a '$' prefix.
 7225 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7226 //               to within an encode class as $primary, $secondary, and $tertiary
 7227 //               rrspectively.  The primary opcode is commonly used to
 7228 //               indicate the type of machine instruction, while secondary
 7229 //               and tertiary are often used for prefix options or addressing
 7230 //               modes.
 7231 // ins_encode -- A list of encode classes with parameters. The encode class
 7232 //               name must have been defined in an 'enc_class' specification
 7233 //               in the encode section of the architecture description.
 7234 
 7235 // ============================================================================
 7236 // Memory (Load/Store) Instructions
 7237 
 7238 // Load Instructions
 7239 
 7240 // Load Byte (8 bit signed)
 7241 instruct loadB(iRegINoSp dst, memory1 mem)
 7242 %{
 7243   match(Set dst (LoadB mem));
 7244   predicate(!needs_acquiring_load(n));
 7245 
 7246   ins_cost(4 * INSN_COST);
 7247   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7248 
 7249   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7250 
 7251   ins_pipe(iload_reg_mem);
 7252 %}
 7253 
 7254 // Load Byte (8 bit signed) into long
 7255 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7256 %{
 7257   match(Set dst (ConvI2L (LoadB mem)));
 7258   predicate(!needs_acquiring_load(n->in(1)));
 7259 
 7260   ins_cost(4 * INSN_COST);
 7261   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7262 
 7263   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7264 
 7265   ins_pipe(iload_reg_mem);
 7266 %}
 7267 
 7268 // Load Byte (8 bit unsigned)
 7269 instruct loadUB(iRegINoSp dst, memory1 mem)
 7270 %{
 7271   match(Set dst (LoadUB mem));
 7272   predicate(!needs_acquiring_load(n));
 7273 
 7274   ins_cost(4 * INSN_COST);
 7275   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7276 
 7277   ins_encode(aarch64_enc_ldrb(dst, mem));
 7278 
 7279   ins_pipe(iload_reg_mem);
 7280 %}
 7281 
 7282 // Load Byte (8 bit unsigned) into long
 7283 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7284 %{
 7285   match(Set dst (ConvI2L (LoadUB mem)));
 7286   predicate(!needs_acquiring_load(n->in(1)));
 7287 
 7288   ins_cost(4 * INSN_COST);
 7289   format %{ "ldrb  $dst, $mem\t# byte" %}
 7290 
 7291   ins_encode(aarch64_enc_ldrb(dst, mem));
 7292 
 7293   ins_pipe(iload_reg_mem);
 7294 %}
 7295 
 7296 // Load Short (16 bit signed)
 7297 instruct loadS(iRegINoSp dst, memory2 mem)
 7298 %{
 7299   match(Set dst (LoadS mem));
 7300   predicate(!needs_acquiring_load(n));
 7301 
 7302   ins_cost(4 * INSN_COST);
 7303   format %{ "ldrshw  $dst, $mem\t# short" %}
 7304 
 7305   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7306 
 7307   ins_pipe(iload_reg_mem);
 7308 %}
 7309 
 7310 // Load Short (16 bit signed) into long
 7311 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7312 %{
 7313   match(Set dst (ConvI2L (LoadS mem)));
 7314   predicate(!needs_acquiring_load(n->in(1)));
 7315 
 7316   ins_cost(4 * INSN_COST);
 7317   format %{ "ldrsh  $dst, $mem\t# short" %}
 7318 
 7319   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7320 
 7321   ins_pipe(iload_reg_mem);
 7322 %}
 7323 
 7324 // Load Char (16 bit unsigned)
 7325 instruct loadUS(iRegINoSp dst, memory2 mem)
 7326 %{
 7327   match(Set dst (LoadUS mem));
 7328   predicate(!needs_acquiring_load(n));
 7329 
 7330   ins_cost(4 * INSN_COST);
 7331   format %{ "ldrh  $dst, $mem\t# short" %}
 7332 
 7333   ins_encode(aarch64_enc_ldrh(dst, mem));
 7334 
 7335   ins_pipe(iload_reg_mem);
 7336 %}
 7337 
 7338 // Load Short/Char (16 bit unsigned) into long
 7339 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7340 %{
 7341   match(Set dst (ConvI2L (LoadUS mem)));
 7342   predicate(!needs_acquiring_load(n->in(1)));
 7343 
 7344   ins_cost(4 * INSN_COST);
 7345   format %{ "ldrh  $dst, $mem\t# short" %}
 7346 
 7347   ins_encode(aarch64_enc_ldrh(dst, mem));
 7348 
 7349   ins_pipe(iload_reg_mem);
 7350 %}
 7351 
 7352 // Load Integer (32 bit signed)
 7353 instruct loadI(iRegINoSp dst, memory4 mem)
 7354 %{
 7355   match(Set dst (LoadI mem));
 7356   predicate(!needs_acquiring_load(n));
 7357 
 7358   ins_cost(4 * INSN_COST);
 7359   format %{ "ldrw  $dst, $mem\t# int" %}
 7360 
 7361   ins_encode(aarch64_enc_ldrw(dst, mem));
 7362 
 7363   ins_pipe(iload_reg_mem);
 7364 %}
 7365 
 7366 // Load Integer (32 bit signed) into long
 7367 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7368 %{
 7369   match(Set dst (ConvI2L (LoadI mem)));
 7370   predicate(!needs_acquiring_load(n->in(1)));
 7371 
 7372   ins_cost(4 * INSN_COST);
 7373   format %{ "ldrsw  $dst, $mem\t# int" %}
 7374 
 7375   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7376 
 7377   ins_pipe(iload_reg_mem);
 7378 %}
 7379 
 7380 // Load Integer (32 bit unsigned) into long
 7381 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7382 %{
 7383   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7384   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7385 
 7386   ins_cost(4 * INSN_COST);
 7387   format %{ "ldrw  $dst, $mem\t# int" %}
 7388 
 7389   ins_encode(aarch64_enc_ldrw(dst, mem));
 7390 
 7391   ins_pipe(iload_reg_mem);
 7392 %}
 7393 
 7394 // Load Long (64 bit signed)
 7395 instruct loadL(iRegLNoSp dst, memory8 mem)
 7396 %{
 7397   match(Set dst (LoadL mem));
 7398   predicate(!needs_acquiring_load(n));
 7399 
 7400   ins_cost(4 * INSN_COST);
 7401   format %{ "ldr  $dst, $mem\t# int" %}
 7402 
 7403   ins_encode(aarch64_enc_ldr(dst, mem));
 7404 
 7405   ins_pipe(iload_reg_mem);
 7406 %}
 7407 
 7408 // Load Range
 7409 instruct loadRange(iRegINoSp dst, memory4 mem)
 7410 %{
 7411   match(Set dst (LoadRange mem));
 7412 
 7413   ins_cost(4 * INSN_COST);
 7414   format %{ "ldrw  $dst, $mem\t# range" %}
 7415 
 7416   ins_encode(aarch64_enc_ldrw(dst, mem));
 7417 
 7418   ins_pipe(iload_reg_mem);
 7419 %}
 7420 
 7421 // Load Pointer
 7422 instruct loadP(iRegPNoSp dst, memory8 mem)
 7423 %{
 7424   match(Set dst (LoadP mem));
 7425   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7426 
 7427   ins_cost(4 * INSN_COST);
 7428   format %{ "ldr  $dst, $mem\t# ptr" %}
 7429 
 7430   ins_encode(aarch64_enc_ldr(dst, mem));
 7431 
 7432   ins_pipe(iload_reg_mem);
 7433 %}
 7434 
 7435 // Load Compressed Pointer
 7436 instruct loadN(iRegNNoSp dst, memory4 mem)
 7437 %{
 7438   match(Set dst (LoadN mem));
 7439   predicate(!needs_acquiring_load(n));
 7440 
 7441   ins_cost(4 * INSN_COST);
 7442   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7443 
 7444   ins_encode(aarch64_enc_ldrw(dst, mem));
 7445 
 7446   ins_pipe(iload_reg_mem);
 7447 %}
 7448 
 7449 // Load Klass Pointer
 7450 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7451 %{
 7452   match(Set dst (LoadKlass mem));
 7453   predicate(!needs_acquiring_load(n));
 7454 
 7455   ins_cost(4 * INSN_COST);
 7456   format %{ "ldr  $dst, $mem\t# class" %}
 7457 
 7458   ins_encode(aarch64_enc_ldr(dst, mem));
 7459 
 7460   ins_pipe(iload_reg_mem);
 7461 %}
 7462 
 7463 // Load Narrow Klass Pointer
 7464 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7465 %{
 7466   match(Set dst (LoadNKlass mem));
 7467   predicate(!needs_acquiring_load(n));
 7468 
 7469   ins_cost(4 * INSN_COST);
 7470   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7471 
 7472   ins_encode(aarch64_enc_ldrw(dst, mem));
 7473 
 7474   ins_pipe(iload_reg_mem);
 7475 %}
 7476 
 7477 // Load Float
 7478 instruct loadF(vRegF dst, memory4 mem)
 7479 %{
 7480   match(Set dst (LoadF mem));
 7481   predicate(!needs_acquiring_load(n));
 7482 
 7483   ins_cost(4 * INSN_COST);
 7484   format %{ "ldrs  $dst, $mem\t# float" %}
 7485 
 7486   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7487 
 7488   ins_pipe(pipe_class_memory);
 7489 %}
 7490 
 7491 // Load Double
 7492 instruct loadD(vRegD dst, memory8 mem)
 7493 %{
 7494   match(Set dst (LoadD mem));
 7495   predicate(!needs_acquiring_load(n));
 7496 
 7497   ins_cost(4 * INSN_COST);
 7498   format %{ "ldrd  $dst, $mem\t# double" %}
 7499 
 7500   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7501 
 7502   ins_pipe(pipe_class_memory);
 7503 %}
 7504 
 7505 
 7506 // Load Int Constant
 7507 instruct loadConI(iRegINoSp dst, immI src)
 7508 %{
 7509   match(Set dst src);
 7510 
 7511   ins_cost(INSN_COST);
 7512   format %{ "mov $dst, $src\t# int" %}
 7513 
 7514   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7515 
 7516   ins_pipe(ialu_imm);
 7517 %}
 7518 
 7519 // Load Long Constant
 7520 instruct loadConL(iRegLNoSp dst, immL src)
 7521 %{
 7522   match(Set dst src);
 7523 
 7524   ins_cost(INSN_COST);
 7525   format %{ "mov $dst, $src\t# long" %}
 7526 
 7527   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7528 
 7529   ins_pipe(ialu_imm);
 7530 %}
 7531 
 7532 // Load Pointer Constant
 7533 
 7534 instruct loadConP(iRegPNoSp dst, immP con)
 7535 %{
 7536   match(Set dst con);
 7537 
 7538   ins_cost(INSN_COST * 4);
 7539   format %{
 7540     "mov  $dst, $con\t# ptr"
 7541   %}
 7542 
 7543   ins_encode(aarch64_enc_mov_p(dst, con));
 7544 
 7545   ins_pipe(ialu_imm);
 7546 %}
 7547 
 7548 // Load Null Pointer Constant
 7549 
 7550 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7551 %{
 7552   match(Set dst con);
 7553 
 7554   ins_cost(INSN_COST);
 7555   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7556 
 7557   ins_encode(aarch64_enc_mov_p0(dst, con));
 7558 
 7559   ins_pipe(ialu_imm);
 7560 %}
 7561 
 7562 // Load Pointer Constant One
 7563 
 7564 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7565 %{
 7566   match(Set dst con);
 7567 
 7568   ins_cost(INSN_COST);
 7569   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7570 
 7571   ins_encode(aarch64_enc_mov_p1(dst, con));
 7572 
 7573   ins_pipe(ialu_imm);
 7574 %}
 7575 
 7576 // Load Byte Map Base Constant
 7577 
 7578 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7579 %{
 7580   match(Set dst con);
 7581 
 7582   ins_cost(INSN_COST);
 7583   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7584 
 7585   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7586 
 7587   ins_pipe(ialu_imm);
 7588 %}
 7589 
 7590 // Load Narrow Pointer Constant
 7591 
 7592 instruct loadConN(iRegNNoSp dst, immN con)
 7593 %{
 7594   match(Set dst con);
 7595 
 7596   ins_cost(INSN_COST * 4);
 7597   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7598 
 7599   ins_encode(aarch64_enc_mov_n(dst, con));
 7600 
 7601   ins_pipe(ialu_imm);
 7602 %}
 7603 
 7604 // Load Narrow Null Pointer Constant
 7605 
 7606 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7607 %{
 7608   match(Set dst con);
 7609 
 7610   ins_cost(INSN_COST);
 7611   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7612 
 7613   ins_encode(aarch64_enc_mov_n0(dst, con));
 7614 
 7615   ins_pipe(ialu_imm);
 7616 %}
 7617 
 7618 // Load Narrow Klass Constant
 7619 
 7620 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7621 %{
 7622   match(Set dst con);
 7623 
 7624   ins_cost(INSN_COST);
 7625   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7626 
 7627   ins_encode(aarch64_enc_mov_nk(dst, con));
 7628 
 7629   ins_pipe(ialu_imm);
 7630 %}
 7631 
 7632 // Load Packed Float Constant
 7633 
 7634 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7635   match(Set dst con);
 7636   ins_cost(INSN_COST * 4);
 7637   format %{ "fmovs  $dst, $con"%}
 7638   ins_encode %{
 7639     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7640   %}
 7641 
 7642   ins_pipe(fp_imm_s);
 7643 %}
 7644 
 7645 // Load Float Constant
 7646 
 7647 instruct loadConF(vRegF dst, immF con) %{
 7648   match(Set dst con);
 7649 
 7650   ins_cost(INSN_COST * 4);
 7651 
 7652   format %{
 7653     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7654   %}
 7655 
 7656   ins_encode %{
 7657     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7658   %}
 7659 
 7660   ins_pipe(fp_load_constant_s);
 7661 %}
 7662 
 7663 // Load Packed Double Constant
 7664 
 7665 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7666   match(Set dst con);
 7667   ins_cost(INSN_COST);
 7668   format %{ "fmovd  $dst, $con"%}
 7669   ins_encode %{
 7670     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7671   %}
 7672 
 7673   ins_pipe(fp_imm_d);
 7674 %}
 7675 
 7676 // Load Double Constant
 7677 
 7678 instruct loadConD(vRegD dst, immD con) %{
 7679   match(Set dst con);
 7680 
 7681   ins_cost(INSN_COST * 5);
 7682   format %{
 7683     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7684   %}
 7685 
 7686   ins_encode %{
 7687     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7688   %}
 7689 
 7690   ins_pipe(fp_load_constant_d);
 7691 %}
 7692 
 7693 // Store Instructions
 7694 
 7695 // Store CMS card-mark Immediate
 7696 instruct storeimmCM0(immI0 zero, memory1 mem)
 7697 %{
 7698   match(Set mem (StoreCM mem zero));
 7699 
 7700   ins_cost(INSN_COST);
 7701   format %{ "storestore (elided)\n\t"
 7702             "strb zr, $mem\t# byte" %}
 7703 
 7704   ins_encode(aarch64_enc_strb0(mem));
 7705 
 7706   ins_pipe(istore_mem);
 7707 %}
 7708 
 7709 // Store CMS card-mark Immediate with intervening StoreStore
 7710 // needed when using CMS with no conditional card marking
 7711 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7712 %{
 7713   match(Set mem (StoreCM mem zero));
 7714 
 7715   ins_cost(INSN_COST * 2);
 7716   format %{ "storestore\n\t"
 7717             "dmb ishst"
 7718             "\n\tstrb zr, $mem\t# byte" %}
 7719 
 7720   ins_encode(aarch64_enc_strb0_ordered(mem));
 7721 
 7722   ins_pipe(istore_mem);
 7723 %}
 7724 
 7725 // Store Byte
 7726 instruct storeB(iRegIorL2I src, memory1 mem)
 7727 %{
 7728   match(Set mem (StoreB mem src));
 7729   predicate(!needs_releasing_store(n));
 7730 
 7731   ins_cost(INSN_COST);
 7732   format %{ "strb  $src, $mem\t# byte" %}
 7733 
 7734   ins_encode(aarch64_enc_strb(src, mem));
 7735 
 7736   ins_pipe(istore_reg_mem);
 7737 %}
 7738 
 7739 
 7740 instruct storeimmB0(immI0 zero, memory1 mem)
 7741 %{
 7742   match(Set mem (StoreB mem zero));
 7743   predicate(!needs_releasing_store(n));
 7744 
 7745   ins_cost(INSN_COST);
 7746   format %{ "strb rscractch2, $mem\t# byte" %}
 7747 
 7748   ins_encode(aarch64_enc_strb0(mem));
 7749 
 7750   ins_pipe(istore_mem);
 7751 %}
 7752 
 7753 // Store Char/Short
 7754 instruct storeC(iRegIorL2I src, memory2 mem)
 7755 %{
 7756   match(Set mem (StoreC mem src));
 7757   predicate(!needs_releasing_store(n));
 7758 
 7759   ins_cost(INSN_COST);
 7760   format %{ "strh  $src, $mem\t# short" %}
 7761 
 7762   ins_encode(aarch64_enc_strh(src, mem));
 7763 
 7764   ins_pipe(istore_reg_mem);
 7765 %}
 7766 
 7767 instruct storeimmC0(immI0 zero, memory2 mem)
 7768 %{
 7769   match(Set mem (StoreC mem zero));
 7770   predicate(!needs_releasing_store(n));
 7771 
 7772   ins_cost(INSN_COST);
 7773   format %{ "strh  zr, $mem\t# short" %}
 7774 
 7775   ins_encode(aarch64_enc_strh0(mem));
 7776 
 7777   ins_pipe(istore_mem);
 7778 %}
 7779 
 7780 // Store Integer
 7781 
 7782 instruct storeI(iRegIorL2I src, memory4 mem)
 7783 %{
 7784   match(Set mem(StoreI mem src));
 7785   predicate(!needs_releasing_store(n));
 7786 
 7787   ins_cost(INSN_COST);
 7788   format %{ "strw  $src, $mem\t# int" %}
 7789 
 7790   ins_encode(aarch64_enc_strw(src, mem));
 7791 
 7792   ins_pipe(istore_reg_mem);
 7793 %}
 7794 
 7795 instruct storeimmI0(immI0 zero, memory4 mem)
 7796 %{
 7797   match(Set mem(StoreI mem zero));
 7798   predicate(!needs_releasing_store(n));
 7799 
 7800   ins_cost(INSN_COST);
 7801   format %{ "strw  zr, $mem\t# int" %}
 7802 
 7803   ins_encode(aarch64_enc_strw0(mem));
 7804 
 7805   ins_pipe(istore_mem);
 7806 %}
 7807 
 7808 // Store Long (64 bit signed)
 7809 instruct storeL(iRegL src, memory8 mem)
 7810 %{
 7811   match(Set mem (StoreL mem src));
 7812   predicate(!needs_releasing_store(n));
 7813 
 7814   ins_cost(INSN_COST);
 7815   format %{ "str  $src, $mem\t# int" %}
 7816 
 7817   ins_encode(aarch64_enc_str(src, mem));
 7818 
 7819   ins_pipe(istore_reg_mem);
 7820 %}
 7821 
 7822 // Store Long (64 bit signed)
 7823 instruct storeimmL0(immL0 zero, memory8 mem)
 7824 %{
 7825   match(Set mem (StoreL mem zero));
 7826   predicate(!needs_releasing_store(n));
 7827 
 7828   ins_cost(INSN_COST);
 7829   format %{ "str  zr, $mem\t# int" %}
 7830 
 7831   ins_encode(aarch64_enc_str0(mem));
 7832 
 7833   ins_pipe(istore_mem);
 7834 %}
 7835 
 7836 // Store Pointer
 7837 instruct storeP(iRegP src, memory8 mem)
 7838 %{
 7839   match(Set mem (StoreP mem src));
 7840   predicate(!needs_releasing_store(n));
 7841 
 7842   ins_cost(INSN_COST);
 7843   format %{ "str  $src, $mem\t# ptr" %}
 7844 
 7845   ins_encode(aarch64_enc_str(src, mem));
 7846 
 7847   ins_pipe(istore_reg_mem);
 7848 %}
 7849 
 7850 // Store Pointer
 7851 instruct storeimmP0(immP0 zero, memory8 mem)
 7852 %{
 7853   match(Set mem (StoreP mem zero));
 7854   predicate(!needs_releasing_store(n));
 7855 
 7856   ins_cost(INSN_COST);
 7857   format %{ "str zr, $mem\t# ptr" %}
 7858 
 7859   ins_encode(aarch64_enc_str0(mem));
 7860 
 7861   ins_pipe(istore_mem);
 7862 %}
 7863 
 7864 // Store Compressed Pointer
 7865 instruct storeN(iRegN src, memory4 mem)
 7866 %{
 7867   match(Set mem (StoreN mem src));
 7868   predicate(!needs_releasing_store(n));
 7869 
 7870   ins_cost(INSN_COST);
 7871   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7872 
 7873   ins_encode(aarch64_enc_strw(src, mem));
 7874 
 7875   ins_pipe(istore_reg_mem);
 7876 %}
 7877 
 7878 instruct storeImmN0(immN0 zero, memory4 mem)
 7879 %{
 7880   match(Set mem (StoreN mem zero));
 7881   predicate(!needs_releasing_store(n));
 7882 
 7883   ins_cost(INSN_COST);
 7884   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7885 
 7886   ins_encode(aarch64_enc_strw0(mem));
 7887 
 7888   ins_pipe(istore_mem);
 7889 %}
 7890 
 7891 // Store Float
 7892 instruct storeF(vRegF src, memory4 mem)
 7893 %{
 7894   match(Set mem (StoreF mem src));
 7895   predicate(!needs_releasing_store(n));
 7896 
 7897   ins_cost(INSN_COST);
 7898   format %{ "strs  $src, $mem\t# float" %}
 7899 
 7900   ins_encode( aarch64_enc_strs(src, mem) );
 7901 
 7902   ins_pipe(pipe_class_memory);
 7903 %}
 7904 
 7905 // TODO
 7906 // implement storeImmF0 and storeFImmPacked
 7907 
 7908 // Store Double
 7909 instruct storeD(vRegD src, memory8 mem)
 7910 %{
 7911   match(Set mem (StoreD mem src));
 7912   predicate(!needs_releasing_store(n));
 7913 
 7914   ins_cost(INSN_COST);
 7915   format %{ "strd  $src, $mem\t# double" %}
 7916 
 7917   ins_encode( aarch64_enc_strd(src, mem) );
 7918 
 7919   ins_pipe(pipe_class_memory);
 7920 %}
 7921 
 7922 // Store Compressed Klass Pointer
 7923 instruct storeNKlass(iRegN src, memory4 mem)
 7924 %{
 7925   predicate(!needs_releasing_store(n));
 7926   match(Set mem (StoreNKlass mem src));
 7927 
 7928   ins_cost(INSN_COST);
 7929   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7930 
 7931   ins_encode(aarch64_enc_strw(src, mem));
 7932 
 7933   ins_pipe(istore_reg_mem);
 7934 %}
 7935 
 7936 // TODO
 7937 // implement storeImmD0 and storeDImmPacked
 7938 
 7939 // prefetch instructions
 7940 // Must be safe to execute with invalid address (cannot fault).
 7941 
 7942 instruct prefetchalloc( memory8 mem ) %{
 7943   match(PrefetchAllocation mem);
 7944 
 7945   ins_cost(INSN_COST);
 7946   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7947 
 7948   ins_encode( aarch64_enc_prefetchw(mem) );
 7949 
 7950   ins_pipe(iload_prefetch);
 7951 %}
 7952 
 7953 //  ---------------- volatile loads and stores ----------------
 7954 
 7955 // Load Byte (8 bit signed)
 7956 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7957 %{
 7958   match(Set dst (LoadB mem));
 7959 
 7960   ins_cost(VOLATILE_REF_COST);
 7961   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7962 
 7963   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7964 
 7965   ins_pipe(pipe_serial);
 7966 %}
 7967 
 7968 // Load Byte (8 bit signed) into long
 7969 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7970 %{
 7971   match(Set dst (ConvI2L (LoadB mem)));
 7972 
 7973   ins_cost(VOLATILE_REF_COST);
 7974   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7975 
 7976   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7977 
 7978   ins_pipe(pipe_serial);
 7979 %}
 7980 
 7981 // Load Byte (8 bit unsigned)
 7982 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7983 %{
 7984   match(Set dst (LoadUB mem));
 7985 
 7986   ins_cost(VOLATILE_REF_COST);
 7987   format %{ "ldarb  $dst, $mem\t# byte" %}
 7988 
 7989   ins_encode(aarch64_enc_ldarb(dst, mem));
 7990 
 7991   ins_pipe(pipe_serial);
 7992 %}
 7993 
 7994 // Load Byte (8 bit unsigned) into long
 7995 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7996 %{
 7997   match(Set dst (ConvI2L (LoadUB mem)));
 7998 
 7999   ins_cost(VOLATILE_REF_COST);
 8000   format %{ "ldarb  $dst, $mem\t# byte" %}
 8001 
 8002   ins_encode(aarch64_enc_ldarb(dst, mem));
 8003 
 8004   ins_pipe(pipe_serial);
 8005 %}
 8006 
 8007 // Load Short (16 bit signed)
 8008 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8009 %{
 8010   match(Set dst (LoadS mem));
 8011 
 8012   ins_cost(VOLATILE_REF_COST);
 8013   format %{ "ldarshw  $dst, $mem\t# short" %}
 8014 
 8015   ins_encode(aarch64_enc_ldarshw(dst, mem));
 8016 
 8017   ins_pipe(pipe_serial);
 8018 %}
 8019 
 8020 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8021 %{
 8022   match(Set dst (LoadUS mem));
 8023 
 8024   ins_cost(VOLATILE_REF_COST);
 8025   format %{ "ldarhw  $dst, $mem\t# short" %}
 8026 
 8027   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8028 
 8029   ins_pipe(pipe_serial);
 8030 %}
 8031 
 8032 // Load Short/Char (16 bit unsigned) into long
 8033 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8034 %{
 8035   match(Set dst (ConvI2L (LoadUS mem)));
 8036 
 8037   ins_cost(VOLATILE_REF_COST);
 8038   format %{ "ldarh  $dst, $mem\t# short" %}
 8039 
 8040   ins_encode(aarch64_enc_ldarh(dst, mem));
 8041 
 8042   ins_pipe(pipe_serial);
 8043 %}
 8044 
 8045 // Load Short/Char (16 bit signed) into long
 8046 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8047 %{
 8048   match(Set dst (ConvI2L (LoadS mem)));
 8049 
 8050   ins_cost(VOLATILE_REF_COST);
 8051   format %{ "ldarh  $dst, $mem\t# short" %}
 8052 
 8053   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8054 
 8055   ins_pipe(pipe_serial);
 8056 %}
 8057 
 8058 // Load Integer (32 bit signed)
 8059 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8060 %{
 8061   match(Set dst (LoadI mem));
 8062 
 8063   ins_cost(VOLATILE_REF_COST);
 8064   format %{ "ldarw  $dst, $mem\t# int" %}
 8065 
 8066   ins_encode(aarch64_enc_ldarw(dst, mem));
 8067 
 8068   ins_pipe(pipe_serial);
 8069 %}
 8070 
 8071 // Load Integer (32 bit unsigned) into long
 8072 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8073 %{
 8074   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8075 
 8076   ins_cost(VOLATILE_REF_COST);
 8077   format %{ "ldarw  $dst, $mem\t# int" %}
 8078 
 8079   ins_encode(aarch64_enc_ldarw(dst, mem));
 8080 
 8081   ins_pipe(pipe_serial);
 8082 %}
 8083 
 8084 // Load Long (64 bit signed)
 8085 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8086 %{
 8087   match(Set dst (LoadL mem));
 8088 
 8089   ins_cost(VOLATILE_REF_COST);
 8090   format %{ "ldar  $dst, $mem\t# int" %}
 8091 
 8092   ins_encode(aarch64_enc_ldar(dst, mem));
 8093 
 8094   ins_pipe(pipe_serial);
 8095 %}
 8096 
 8097 // Load Pointer
 8098 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8099 %{
 8100   match(Set dst (LoadP mem));
 8101   predicate(n->as_Load()->barrier_data() == 0);
 8102 
 8103   ins_cost(VOLATILE_REF_COST);
 8104   format %{ "ldar  $dst, $mem\t# ptr" %}
 8105 
 8106   ins_encode(aarch64_enc_ldar(dst, mem));
 8107 
 8108   ins_pipe(pipe_serial);
 8109 %}
 8110 
 8111 // Load Compressed Pointer
 8112 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8113 %{
 8114   match(Set dst (LoadN mem));
 8115 
 8116   ins_cost(VOLATILE_REF_COST);
 8117   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8118 
 8119   ins_encode(aarch64_enc_ldarw(dst, mem));
 8120 
 8121   ins_pipe(pipe_serial);
 8122 %}
 8123 
 8124 // Load Float
 8125 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8126 %{
 8127   match(Set dst (LoadF mem));
 8128 
 8129   ins_cost(VOLATILE_REF_COST);
 8130   format %{ "ldars  $dst, $mem\t# float" %}
 8131 
 8132   ins_encode( aarch64_enc_fldars(dst, mem) );
 8133 
 8134   ins_pipe(pipe_serial);
 8135 %}
 8136 
 8137 // Load Double
 8138 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8139 %{
 8140   match(Set dst (LoadD mem));
 8141 
 8142   ins_cost(VOLATILE_REF_COST);
 8143   format %{ "ldard  $dst, $mem\t# double" %}
 8144 
 8145   ins_encode( aarch64_enc_fldard(dst, mem) );
 8146 
 8147   ins_pipe(pipe_serial);
 8148 %}
 8149 
 8150 // Store Byte
 8151 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8152 %{
 8153   match(Set mem (StoreB mem src));
 8154 
 8155   ins_cost(VOLATILE_REF_COST);
 8156   format %{ "stlrb  $src, $mem\t# byte" %}
 8157 
 8158   ins_encode(aarch64_enc_stlrb(src, mem));
 8159 
 8160   ins_pipe(pipe_class_memory);
 8161 %}
 8162 
 8163 // Store Char/Short
 8164 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8165 %{
 8166   match(Set mem (StoreC mem src));
 8167 
 8168   ins_cost(VOLATILE_REF_COST);
 8169   format %{ "stlrh  $src, $mem\t# short" %}
 8170 
 8171   ins_encode(aarch64_enc_stlrh(src, mem));
 8172 
 8173   ins_pipe(pipe_class_memory);
 8174 %}
 8175 
 8176 // Store Integer
 8177 
 8178 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8179 %{
 8180   match(Set mem(StoreI mem src));
 8181 
 8182   ins_cost(VOLATILE_REF_COST);
 8183   format %{ "stlrw  $src, $mem\t# int" %}
 8184 
 8185   ins_encode(aarch64_enc_stlrw(src, mem));
 8186 
 8187   ins_pipe(pipe_class_memory);
 8188 %}
 8189 
 8190 // Store Long (64 bit signed)
 8191 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8192 %{
 8193   match(Set mem (StoreL mem src));
 8194 
 8195   ins_cost(VOLATILE_REF_COST);
 8196   format %{ "stlr  $src, $mem\t# int" %}
 8197 
 8198   ins_encode(aarch64_enc_stlr(src, mem));
 8199 
 8200   ins_pipe(pipe_class_memory);
 8201 %}
 8202 
 8203 // Store Pointer
 8204 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8205 %{
 8206   match(Set mem (StoreP mem src));
 8207 
 8208   ins_cost(VOLATILE_REF_COST);
 8209   format %{ "stlr  $src, $mem\t# ptr" %}
 8210 
 8211   ins_encode(aarch64_enc_stlr(src, mem));
 8212 
 8213   ins_pipe(pipe_class_memory);
 8214 %}
 8215 
 8216 // Store Compressed Pointer
 8217 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8218 %{
 8219   match(Set mem (StoreN mem src));
 8220 
 8221   ins_cost(VOLATILE_REF_COST);
 8222   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8223 
 8224   ins_encode(aarch64_enc_stlrw(src, mem));
 8225 
 8226   ins_pipe(pipe_class_memory);
 8227 %}
 8228 
 8229 // Store Float
 8230 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8231 %{
 8232   match(Set mem (StoreF mem src));
 8233 
 8234   ins_cost(VOLATILE_REF_COST);
 8235   format %{ "stlrs  $src, $mem\t# float" %}
 8236 
 8237   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8238 
 8239   ins_pipe(pipe_class_memory);
 8240 %}
 8241 
 8242 // TODO
 8243 // implement storeImmF0 and storeFImmPacked
 8244 
 8245 // Store Double
 8246 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8247 %{
 8248   match(Set mem (StoreD mem src));
 8249 
 8250   ins_cost(VOLATILE_REF_COST);
 8251   format %{ "stlrd  $src, $mem\t# double" %}
 8252 
 8253   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8254 
 8255   ins_pipe(pipe_class_memory);
 8256 %}
 8257 
 8258 //  ---------------- end of volatile loads and stores ----------------
 8259 
 8260 instruct cacheWB(indirect addr)
 8261 %{
 8262   predicate(VM_Version::supports_data_cache_line_flush());
 8263   match(CacheWB addr);
 8264 
 8265   ins_cost(100);
 8266   format %{"cache wb $addr" %}
 8267   ins_encode %{
 8268     assert($addr->index_position() < 0, "should be");
 8269     assert($addr$$disp == 0, "should be");
 8270     __ cache_wb(Address($addr$$base$$Register, 0));
 8271   %}
 8272   ins_pipe(pipe_slow); // XXX
 8273 %}
 8274 
 8275 instruct cacheWBPreSync()
 8276 %{
 8277   predicate(VM_Version::supports_data_cache_line_flush());
 8278   match(CacheWBPreSync);
 8279 
 8280   ins_cost(100);
 8281   format %{"cache wb presync" %}
 8282   ins_encode %{
 8283     __ cache_wbsync(true);
 8284   %}
 8285   ins_pipe(pipe_slow); // XXX
 8286 %}
 8287 
 8288 instruct cacheWBPostSync()
 8289 %{
 8290   predicate(VM_Version::supports_data_cache_line_flush());
 8291   match(CacheWBPostSync);
 8292 
 8293   ins_cost(100);
 8294   format %{"cache wb postsync" %}
 8295   ins_encode %{
 8296     __ cache_wbsync(false);
 8297   %}
 8298   ins_pipe(pipe_slow); // XXX
 8299 %}
 8300 
 8301 // ============================================================================
 8302 // BSWAP Instructions
 8303 
 8304 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8305   match(Set dst (ReverseBytesI src));
 8306 
 8307   ins_cost(INSN_COST);
 8308   format %{ "revw  $dst, $src" %}
 8309 
 8310   ins_encode %{
 8311     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8312   %}
 8313 
 8314   ins_pipe(ialu_reg);
 8315 %}
 8316 
 8317 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8318   match(Set dst (ReverseBytesL src));
 8319 
 8320   ins_cost(INSN_COST);
 8321   format %{ "rev  $dst, $src" %}
 8322 
 8323   ins_encode %{
 8324     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8325   %}
 8326 
 8327   ins_pipe(ialu_reg);
 8328 %}
 8329 
 8330 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8331   match(Set dst (ReverseBytesUS src));
 8332 
 8333   ins_cost(INSN_COST);
 8334   format %{ "rev16w  $dst, $src" %}
 8335 
 8336   ins_encode %{
 8337     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8338   %}
 8339 
 8340   ins_pipe(ialu_reg);
 8341 %}
 8342 
 8343 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8344   match(Set dst (ReverseBytesS src));
 8345 
 8346   ins_cost(INSN_COST);
 8347   format %{ "rev16w  $dst, $src\n\t"
 8348             "sbfmw $dst, $dst, #0, #15" %}
 8349 
 8350   ins_encode %{
 8351     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8352     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8353   %}
 8354 
 8355   ins_pipe(ialu_reg);
 8356 %}
 8357 
 8358 // ============================================================================
 8359 // Zero Count Instructions
 8360 
 8361 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8362   match(Set dst (CountLeadingZerosI src));
 8363 
 8364   ins_cost(INSN_COST);
 8365   format %{ "clzw  $dst, $src" %}
 8366   ins_encode %{
 8367     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8368   %}
 8369 
 8370   ins_pipe(ialu_reg);
 8371 %}
 8372 
 8373 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8374   match(Set dst (CountLeadingZerosL src));
 8375 
 8376   ins_cost(INSN_COST);
 8377   format %{ "clz   $dst, $src" %}
 8378   ins_encode %{
 8379     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8380   %}
 8381 
 8382   ins_pipe(ialu_reg);
 8383 %}
 8384 
 8385 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8386   match(Set dst (CountTrailingZerosI src));
 8387 
 8388   ins_cost(INSN_COST * 2);
 8389   format %{ "rbitw  $dst, $src\n\t"
 8390             "clzw   $dst, $dst" %}
 8391   ins_encode %{
 8392     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8393     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8394   %}
 8395 
 8396   ins_pipe(ialu_reg);
 8397 %}
 8398 
 8399 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8400   match(Set dst (CountTrailingZerosL src));
 8401 
 8402   ins_cost(INSN_COST * 2);
 8403   format %{ "rbit   $dst, $src\n\t"
 8404             "clz    $dst, $dst" %}
 8405   ins_encode %{
 8406     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8407     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8408   %}
 8409 
 8410   ins_pipe(ialu_reg);
 8411 %}
 8412 
 8413 //---------- Population Count Instructions -------------------------------------
 8414 //
 8415 
 8416 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8417   predicate(UsePopCountInstruction);
 8418   match(Set dst (PopCountI src));
 8419   effect(TEMP tmp);
 8420   ins_cost(INSN_COST * 13);
 8421 
 8422   format %{ "movw   $src, $src\n\t"
 8423             "mov    $tmp, $src\t# vector (1D)\n\t"
 8424             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8425             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8426             "mov    $dst, $tmp\t# vector (1D)" %}
 8427   ins_encode %{
 8428     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8429     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8430     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8431     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8432     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8433   %}
 8434 
 8435   ins_pipe(pipe_class_default);
 8436 %}
 8437 
 8438 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8439   predicate(UsePopCountInstruction);
 8440   match(Set dst (PopCountI (LoadI mem)));
 8441   effect(TEMP tmp);
 8442   ins_cost(INSN_COST * 13);
 8443 
 8444   format %{ "ldrs   $tmp, $mem\n\t"
 8445             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8446             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8447             "mov    $dst, $tmp\t# vector (1D)" %}
 8448   ins_encode %{
 8449     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8450     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8451               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8452     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8453     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8454     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8455   %}
 8456 
 8457   ins_pipe(pipe_class_default);
 8458 %}
 8459 
 8460 // Note: Long.bitCount(long) returns an int.
 8461 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8462   predicate(UsePopCountInstruction);
 8463   match(Set dst (PopCountL src));
 8464   effect(TEMP tmp);
 8465   ins_cost(INSN_COST * 13);
 8466 
 8467   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8468             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8469             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8470             "mov    $dst, $tmp\t# vector (1D)" %}
 8471   ins_encode %{
 8472     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8473     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8474     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8475     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8476   %}
 8477 
 8478   ins_pipe(pipe_class_default);
 8479 %}
 8480 
 8481 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8482   predicate(UsePopCountInstruction);
 8483   match(Set dst (PopCountL (LoadL mem)));
 8484   effect(TEMP tmp);
 8485   ins_cost(INSN_COST * 13);
 8486 
 8487   format %{ "ldrd   $tmp, $mem\n\t"
 8488             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8489             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8490             "mov    $dst, $tmp\t# vector (1D)" %}
 8491   ins_encode %{
 8492     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8493     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8494               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8495     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8496     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8497     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8498   %}
 8499 
 8500   ins_pipe(pipe_class_default);
 8501 %}
 8502 
 8503 // ============================================================================
 8504 // MemBar Instruction
 8505 
 8506 instruct load_fence() %{
 8507   match(LoadFence);
 8508   ins_cost(VOLATILE_REF_COST);
 8509 
 8510   format %{ "load_fence" %}
 8511 
 8512   ins_encode %{
 8513     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8514   %}
 8515   ins_pipe(pipe_serial);
 8516 %}
 8517 
 8518 instruct unnecessary_membar_acquire() %{
 8519   predicate(unnecessary_acquire(n));
 8520   match(MemBarAcquire);
 8521   ins_cost(0);
 8522 
 8523   format %{ "membar_acquire (elided)" %}
 8524 
 8525   ins_encode %{
 8526     __ block_comment("membar_acquire (elided)");
 8527   %}
 8528 
 8529   ins_pipe(pipe_class_empty);
 8530 %}
 8531 
 8532 instruct membar_acquire() %{
 8533   match(MemBarAcquire);
 8534   ins_cost(VOLATILE_REF_COST);
 8535 
 8536   format %{ "membar_acquire\n\t"
 8537             "dmb ish" %}
 8538 
 8539   ins_encode %{
 8540     __ block_comment("membar_acquire");
 8541     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8542   %}
 8543 
 8544   ins_pipe(pipe_serial);
 8545 %}
 8546 
 8547 
 8548 instruct membar_acquire_lock() %{
 8549   match(MemBarAcquireLock);
 8550   ins_cost(VOLATILE_REF_COST);
 8551 
 8552   format %{ "membar_acquire_lock (elided)" %}
 8553 
 8554   ins_encode %{
 8555     __ block_comment("membar_acquire_lock (elided)");
 8556   %}
 8557 
 8558   ins_pipe(pipe_serial);
 8559 %}
 8560 
 8561 instruct store_fence() %{
 8562   match(StoreFence);
 8563   ins_cost(VOLATILE_REF_COST);
 8564 
 8565   format %{ "store_fence" %}
 8566 
 8567   ins_encode %{
 8568     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8569   %}
 8570   ins_pipe(pipe_serial);
 8571 %}
 8572 
 8573 instruct unnecessary_membar_release() %{
 8574   predicate(unnecessary_release(n));
 8575   match(MemBarRelease);
 8576   ins_cost(0);
 8577 
 8578   format %{ "membar_release (elided)" %}
 8579 
 8580   ins_encode %{
 8581     __ block_comment("membar_release (elided)");
 8582   %}
 8583   ins_pipe(pipe_serial);
 8584 %}
 8585 
 8586 instruct membar_release() %{
 8587   match(MemBarRelease);
 8588   ins_cost(VOLATILE_REF_COST);
 8589 
 8590   format %{ "membar_release\n\t"
 8591             "dmb ish" %}
 8592 
 8593   ins_encode %{
 8594     __ block_comment("membar_release");
 8595     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8596   %}
 8597   ins_pipe(pipe_serial);
 8598 %}
 8599 
 8600 instruct membar_storestore() %{
 8601   match(MemBarStoreStore);
 8602   ins_cost(VOLATILE_REF_COST);
 8603 
 8604   format %{ "MEMBAR-store-store" %}
 8605 
 8606   ins_encode %{
 8607     __ membar(Assembler::StoreStore);
 8608   %}
 8609   ins_pipe(pipe_serial);
 8610 %}
 8611 
 8612 instruct membar_release_lock() %{
 8613   match(MemBarReleaseLock);
 8614   ins_cost(VOLATILE_REF_COST);
 8615 
 8616   format %{ "membar_release_lock (elided)" %}
 8617 
 8618   ins_encode %{
 8619     __ block_comment("membar_release_lock (elided)");
 8620   %}
 8621 
 8622   ins_pipe(pipe_serial);
 8623 %}
 8624 
 8625 instruct unnecessary_membar_volatile() %{
 8626   predicate(unnecessary_volatile(n));
 8627   match(MemBarVolatile);
 8628   ins_cost(0);
 8629 
 8630   format %{ "membar_volatile (elided)" %}
 8631 
 8632   ins_encode %{
 8633     __ block_comment("membar_volatile (elided)");
 8634   %}
 8635 
 8636   ins_pipe(pipe_serial);
 8637 %}
 8638 
 8639 instruct membar_volatile() %{
 8640   match(MemBarVolatile);
 8641   ins_cost(VOLATILE_REF_COST*100);
 8642 
 8643   format %{ "membar_volatile\n\t"
 8644              "dmb ish"%}
 8645 
 8646   ins_encode %{
 8647     __ block_comment("membar_volatile");
 8648     __ membar(Assembler::StoreLoad);
 8649   %}
 8650 
 8651   ins_pipe(pipe_serial);
 8652 %}
 8653 
 8654 // ============================================================================
 8655 // Cast/Convert Instructions
 8656 
 8657 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8658   match(Set dst (CastX2P src));
 8659 
 8660   ins_cost(INSN_COST);
 8661   format %{ "mov $dst, $src\t# long -> ptr" %}
 8662 
 8663   ins_encode %{
 8664     if ($dst$$reg != $src$$reg) {
 8665       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8666     }
 8667   %}
 8668 
 8669   ins_pipe(ialu_reg);
 8670 %}
 8671 
 8672 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8673   match(Set dst (CastP2X src));
 8674 
 8675   ins_cost(INSN_COST);
 8676   format %{ "mov $dst, $src\t# ptr -> long" %}
 8677 
 8678   ins_encode %{
 8679     if ($dst$$reg != $src$$reg) {
 8680       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8681     }
 8682   %}
 8683 
 8684   ins_pipe(ialu_reg);
 8685 %}
 8686 
 8687 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8688   match(Set dst (CastP2X src));
 8689 
 8690   ins_cost(INSN_COST);
 8691   format %{ "mov $dst, $src\t# ptr -> long" %}
 8692 
 8693   ins_encode %{
 8694     if ($dst$$reg != $src$$reg) {
 8695       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8696     }
 8697   %}
 8698 
 8699   ins_pipe(ialu_reg);
 8700 %}
 8701 
 8702 // Convert oop into int for vectors alignment masking
 8703 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8704   match(Set dst (ConvL2I (CastP2X src)));
 8705 
 8706   ins_cost(INSN_COST);
 8707   format %{ "movw $dst, $src\t# ptr -> int" %}
 8708   ins_encode %{
 8709     __ movw($dst$$Register, $src$$Register);
 8710   %}
 8711 
 8712   ins_pipe(ialu_reg);
 8713 %}
 8714 
 8715 // Convert compressed oop into int for vectors alignment masking
 8716 // in case of 32bit oops (heap < 4Gb).
 8717 instruct convN2I(iRegINoSp dst, iRegN src)
 8718 %{
 8719   predicate(CompressedOops::shift() == 0);
 8720   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8721 
 8722   ins_cost(INSN_COST);
 8723   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8724   ins_encode %{
 8725     __ movw($dst$$Register, $src$$Register);
 8726   %}
 8727 
 8728   ins_pipe(ialu_reg);
 8729 %}
 8730 
 8731 
 8732 // Convert oop pointer into compressed form
 8733 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8734   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8735   match(Set dst (EncodeP src));
 8736   effect(KILL cr);
 8737   ins_cost(INSN_COST * 3);
 8738   format %{ "encode_heap_oop $dst, $src" %}
 8739   ins_encode %{
 8740     Register s = $src$$Register;
 8741     Register d = $dst$$Register;
 8742     __ encode_heap_oop(d, s);
 8743   %}
 8744   ins_pipe(ialu_reg);
 8745 %}
 8746 
 8747 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8748   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8749   match(Set dst (EncodeP src));
 8750   ins_cost(INSN_COST * 3);
 8751   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8752   ins_encode %{
 8753     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8754   %}
 8755   ins_pipe(ialu_reg);
 8756 %}
 8757 
 8758 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8759   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8760             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8761   match(Set dst (DecodeN src));
 8762   ins_cost(INSN_COST * 3);
 8763   format %{ "decode_heap_oop $dst, $src" %}
 8764   ins_encode %{
 8765     Register s = $src$$Register;
 8766     Register d = $dst$$Register;
 8767     __ decode_heap_oop(d, s);
 8768   %}
 8769   ins_pipe(ialu_reg);
 8770 %}
 8771 
 8772 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8773   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8774             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8775   match(Set dst (DecodeN src));
 8776   ins_cost(INSN_COST * 3);
 8777   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8778   ins_encode %{
 8779     Register s = $src$$Register;
 8780     Register d = $dst$$Register;
 8781     __ decode_heap_oop_not_null(d, s);
 8782   %}
 8783   ins_pipe(ialu_reg);
 8784 %}
 8785 
 8786 // n.b. AArch64 implementations of encode_klass_not_null and
 8787 // decode_klass_not_null do not modify the flags register so, unlike
 8788 // Intel, we don't kill CR as a side effect here
 8789 
 8790 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8791   match(Set dst (EncodePKlass src));
 8792 
 8793   ins_cost(INSN_COST * 3);
 8794   format %{ "encode_klass_not_null $dst,$src" %}
 8795 
 8796   ins_encode %{
 8797     Register src_reg = as_Register($src$$reg);
 8798     Register dst_reg = as_Register($dst$$reg);
 8799     __ encode_klass_not_null(dst_reg, src_reg);
 8800   %}
 8801 
 8802    ins_pipe(ialu_reg);
 8803 %}
 8804 
 8805 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8806   match(Set dst (DecodeNKlass src));
 8807 
 8808   ins_cost(INSN_COST * 3);
 8809   format %{ "decode_klass_not_null $dst,$src" %}
 8810 
 8811   ins_encode %{
 8812     Register src_reg = as_Register($src$$reg);
 8813     Register dst_reg = as_Register($dst$$reg);
 8814     if (dst_reg != src_reg) {
 8815       __ decode_klass_not_null(dst_reg, src_reg);
 8816     } else {
 8817       __ decode_klass_not_null(dst_reg);
 8818     }
 8819   %}
 8820 
 8821    ins_pipe(ialu_reg);
 8822 %}
 8823 
 8824 instruct checkCastPP(iRegPNoSp dst)
 8825 %{
 8826   match(Set dst (CheckCastPP dst));
 8827 
 8828   size(0);
 8829   format %{ "# checkcastPP of $dst" %}
 8830   ins_encode(/* empty encoding */);
 8831   ins_pipe(pipe_class_empty);
 8832 %}
 8833 
 8834 instruct castPP(iRegPNoSp dst)
 8835 %{
 8836   match(Set dst (CastPP dst));
 8837 
 8838   size(0);
 8839   format %{ "# castPP of $dst" %}
 8840   ins_encode(/* empty encoding */);
 8841   ins_pipe(pipe_class_empty);
 8842 %}
 8843 
 8844 instruct castII(iRegI dst)
 8845 %{
 8846   match(Set dst (CastII dst));
 8847 
 8848   size(0);
 8849   format %{ "# castII of $dst" %}
 8850   ins_encode(/* empty encoding */);
 8851   ins_cost(0);
 8852   ins_pipe(pipe_class_empty);
 8853 %}
 8854 
 8855 instruct castLL(iRegL dst)
 8856 %{
 8857   match(Set dst (CastLL dst));
 8858 
 8859   size(0);
 8860   format %{ "# castLL of $dst" %}
 8861   ins_encode(/* empty encoding */);
 8862   ins_cost(0);
 8863   ins_pipe(pipe_class_empty);
 8864 %}
 8865 
 8866 instruct castFF(vRegF dst)
 8867 %{
 8868   match(Set dst (CastFF dst));
 8869 
 8870   size(0);
 8871   format %{ "# castFF of $dst" %}
 8872   ins_encode(/* empty encoding */);
 8873   ins_cost(0);
 8874   ins_pipe(pipe_class_empty);
 8875 %}
 8876 
 8877 instruct castDD(vRegD dst)
 8878 %{
 8879   match(Set dst (CastDD dst));
 8880 
 8881   size(0);
 8882   format %{ "# castDD of $dst" %}
 8883   ins_encode(/* empty encoding */);
 8884   ins_cost(0);
 8885   ins_pipe(pipe_class_empty);
 8886 %}
 8887 
 8888 instruct castVVD(vecD dst)
 8889 %{
 8890   match(Set dst (CastVV dst));
 8891 
 8892   size(0);
 8893   format %{ "# castVV of $dst" %}
 8894   ins_encode(/* empty encoding */);
 8895   ins_cost(0);
 8896   ins_pipe(pipe_class_empty);
 8897 %}
 8898 
 8899 instruct castVVX(vecX dst)
 8900 %{
 8901   match(Set dst (CastVV dst));
 8902 
 8903   size(0);
 8904   format %{ "# castVV of $dst" %}
 8905   ins_encode(/* empty encoding */);
 8906   ins_cost(0);
 8907   ins_pipe(pipe_class_empty);
 8908 %}
 8909 
 8910 instruct castVV(vReg dst)
 8911 %{
 8912   match(Set dst (CastVV dst));
 8913 
 8914   size(0);
 8915   format %{ "# castVV of $dst" %}
 8916   ins_encode(/* empty encoding */);
 8917   ins_cost(0);
 8918   ins_pipe(pipe_class_empty);
 8919 %}
 8920 
 8921 // ============================================================================
 8922 // Atomic operation instructions
 8923 //
 8924 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8925 // Store{PIL}Conditional instructions using a normal load for the
 8926 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8927 //
 8928 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8929 // pair to lock object allocations from Eden space when not using
 8930 // TLABs.
 8931 //
 8932 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8933 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8934 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8935 // only for 64-bit.
 8936 //
 8937 // We implement LoadPLocked and StorePLocked instructions using,
 8938 // respectively the AArch64 hw load-exclusive and store-conditional
 8939 // instructions. Whereas we must implement each of
 8940 // Store{IL}Conditional using a CAS which employs a pair of
 8941 // instructions comprising a load-exclusive followed by a
 8942 // store-conditional.
 8943 
 8944 
 8945 // Locked-load (linked load) of the current heap-top
 8946 // used when updating the eden heap top
 8947 // implemented using ldaxr on AArch64
 8948 
 8949 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 8950 %{
 8951   match(Set dst (LoadPLocked mem));
 8952 
 8953   ins_cost(VOLATILE_REF_COST);
 8954 
 8955   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 8956 
 8957   ins_encode(aarch64_enc_ldaxr(dst, mem));
 8958 
 8959   ins_pipe(pipe_serial);
 8960 %}
 8961 
 8962 // Conditional-store of the updated heap-top.
 8963 // Used during allocation of the shared heap.
 8964 // Sets flag (EQ) on success.
 8965 // implemented using stlxr on AArch64.
 8966 
 8967 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 8968 %{
 8969   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8970 
 8971   ins_cost(VOLATILE_REF_COST);
 8972 
 8973  // TODO
 8974  // do we need to do a store-conditional release or can we just use a
 8975  // plain store-conditional?
 8976 
 8977   format %{
 8978     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 8979     "cmpw rscratch1, zr\t# EQ on successful write"
 8980   %}
 8981 
 8982   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 8983 
 8984   ins_pipe(pipe_serial);
 8985 %}
 8986 
 8987 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 8988 %{
 8989   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 8990 
 8991   ins_cost(VOLATILE_REF_COST);
 8992 
 8993   format %{
 8994     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 8995     "cmpw rscratch1, zr\t# EQ on successful write"
 8996   %}
 8997 
 8998   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 8999 
 9000   ins_pipe(pipe_slow);
 9001 %}
 9002 
 9003 // storeIConditional also has acquire semantics, for no better reason
 9004 // than matching storeLConditional.  At the time of writing this
 9005 // comment storeIConditional was not used anywhere by AArch64.
 9006 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9007 %{
 9008   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9009 
 9010   ins_cost(VOLATILE_REF_COST);
 9011 
 9012   format %{
 9013     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9014     "cmpw rscratch1, zr\t# EQ on successful write"
 9015   %}
 9016 
 9017   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9018 
 9019   ins_pipe(pipe_slow);
 9020 %}
 9021 
 9022 // standard CompareAndSwapX when we are using barriers
 9023 // these have higher priority than the rules selected by a predicate
 9024 
 9025 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9026 // can't match them
 9027 
 9028 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9029 
 9030   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9031   ins_cost(2 * VOLATILE_REF_COST);
 9032 
 9033   effect(KILL cr);
 9034 
 9035   format %{
 9036     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9037     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9038   %}
 9039 
 9040   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9041             aarch64_enc_cset_eq(res));
 9042 
 9043   ins_pipe(pipe_slow);
 9044 %}
 9045 
 9046 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9047 
 9048   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9049   ins_cost(2 * VOLATILE_REF_COST);
 9050 
 9051   effect(KILL cr);
 9052 
 9053   format %{
 9054     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9055     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9056   %}
 9057 
 9058   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9059             aarch64_enc_cset_eq(res));
 9060 
 9061   ins_pipe(pipe_slow);
 9062 %}
 9063 
 9064 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9065 
 9066   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9067   ins_cost(2 * VOLATILE_REF_COST);
 9068 
 9069   effect(KILL cr);
 9070 
 9071  format %{
 9072     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9073     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9074  %}
 9075 
 9076  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9077             aarch64_enc_cset_eq(res));
 9078 
 9079   ins_pipe(pipe_slow);
 9080 %}
 9081 
 9082 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9083 
 9084   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9085   ins_cost(2 * VOLATILE_REF_COST);
 9086 
 9087   effect(KILL cr);
 9088 
 9089  format %{
 9090     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9091     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9092  %}
 9093 
 9094  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9095             aarch64_enc_cset_eq(res));
 9096 
 9097   ins_pipe(pipe_slow);
 9098 %}
 9099 
 9100 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9101 
 9102   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9103   predicate(n->as_LoadStore()->barrier_data() == 0);
 9104   ins_cost(2 * VOLATILE_REF_COST);
 9105 
 9106   effect(KILL cr);
 9107 
 9108  format %{
 9109     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9110     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9111  %}
 9112 
 9113  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9114             aarch64_enc_cset_eq(res));
 9115 
 9116   ins_pipe(pipe_slow);
 9117 %}
 9118 
 9119 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9120 
 9121   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9122   ins_cost(2 * VOLATILE_REF_COST);
 9123 
 9124   effect(KILL cr);
 9125 
 9126  format %{
 9127     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9128     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9129  %}
 9130 
 9131  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9132             aarch64_enc_cset_eq(res));
 9133 
 9134   ins_pipe(pipe_slow);
 9135 %}
 9136 
 9137 // alternative CompareAndSwapX when we are eliding barriers
 9138 
 9139 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9140 
 9141   predicate(needs_acquiring_load_exclusive(n));
 9142   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9143   ins_cost(VOLATILE_REF_COST);
 9144 
 9145   effect(KILL cr);
 9146 
 9147   format %{
 9148     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9149     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9150   %}
 9151 
 9152   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9153             aarch64_enc_cset_eq(res));
 9154 
 9155   ins_pipe(pipe_slow);
 9156 %}
 9157 
 9158 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9159 
 9160   predicate(needs_acquiring_load_exclusive(n));
 9161   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9162   ins_cost(VOLATILE_REF_COST);
 9163 
 9164   effect(KILL cr);
 9165 
 9166   format %{
 9167     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9168     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9169   %}
 9170 
 9171   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9172             aarch64_enc_cset_eq(res));
 9173 
 9174   ins_pipe(pipe_slow);
 9175 %}
 9176 
 9177 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9178 
 9179   predicate(needs_acquiring_load_exclusive(n));
 9180   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9181   ins_cost(VOLATILE_REF_COST);
 9182 
 9183   effect(KILL cr);
 9184 
 9185  format %{
 9186     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9187     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9188  %}
 9189 
 9190  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9191             aarch64_enc_cset_eq(res));
 9192 
 9193   ins_pipe(pipe_slow);
 9194 %}
 9195 
 9196 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9197 
 9198   predicate(needs_acquiring_load_exclusive(n));
 9199   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9200   ins_cost(VOLATILE_REF_COST);
 9201 
 9202   effect(KILL cr);
 9203 
 9204  format %{
 9205     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9206     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9207  %}
 9208 
 9209  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9210             aarch64_enc_cset_eq(res));
 9211 
 9212   ins_pipe(pipe_slow);
 9213 %}
 9214 
 9215 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9216 
 9217   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9218   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9219   ins_cost(VOLATILE_REF_COST);
 9220 
 9221   effect(KILL cr);
 9222 
 9223  format %{
 9224     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9225     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9226  %}
 9227 
 9228  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9229             aarch64_enc_cset_eq(res));
 9230 
 9231   ins_pipe(pipe_slow);
 9232 %}
 9233 
 9234 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9235 
 9236   predicate(needs_acquiring_load_exclusive(n));
 9237   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9238   ins_cost(VOLATILE_REF_COST);
 9239 
 9240   effect(KILL cr);
 9241 
 9242  format %{
 9243     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9244     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9245  %}
 9246 
 9247  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9248             aarch64_enc_cset_eq(res));
 9249 
 9250   ins_pipe(pipe_slow);
 9251 %}
 9252 
 9253 
 9254 // ---------------------------------------------------------------------
 9255 
 9256 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9257 
 9258 // Sundry CAS operations.  Note that release is always true,
 9259 // regardless of the memory ordering of the CAS.  This is because we
 9260 // need the volatile case to be sequentially consistent but there is
 9261 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9262 // can't check the type of memory ordering here, so we always emit a
 9263 // STLXR.
 9264 
 9265 // This section is generated from aarch64_ad_cas.m4
 9266 
 9267 
 9268 
 9269 // This pattern is generated automatically from cas.m4.
 9270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9271 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9272 
 9273   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9274   ins_cost(2 * VOLATILE_REF_COST);
 9275   effect(TEMP_DEF res, KILL cr);
 9276   format %{
 9277     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9278   %}
 9279   ins_encode %{
 9280     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9281                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9282                /*weak*/ false, $res$$Register);
 9283     __ sxtbw($res$$Register, $res$$Register);
 9284   %}
 9285   ins_pipe(pipe_slow);
 9286 %}
 9287 
 9288 // This pattern is generated automatically from cas.m4.
 9289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9290 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9291 
 9292   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9293   ins_cost(2 * VOLATILE_REF_COST);
 9294   effect(TEMP_DEF res, KILL cr);
 9295   format %{
 9296     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9297   %}
 9298   ins_encode %{
 9299     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9300                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9301                /*weak*/ false, $res$$Register);
 9302     __ sxthw($res$$Register, $res$$Register);
 9303   %}
 9304   ins_pipe(pipe_slow);
 9305 %}
 9306 
 9307 // This pattern is generated automatically from cas.m4.
 9308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9309 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9310 
 9311   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9312   ins_cost(2 * VOLATILE_REF_COST);
 9313   effect(TEMP_DEF res, KILL cr);
 9314   format %{
 9315     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9316   %}
 9317   ins_encode %{
 9318     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9319                Assembler::word, /*acquire*/ false, /*release*/ true,
 9320                /*weak*/ false, $res$$Register);
 9321   %}
 9322   ins_pipe(pipe_slow);
 9323 %}
 9324 
 9325 // This pattern is generated automatically from cas.m4.
 9326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9327 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9328 
 9329   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9330   ins_cost(2 * VOLATILE_REF_COST);
 9331   effect(TEMP_DEF res, KILL cr);
 9332   format %{
 9333     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9334   %}
 9335   ins_encode %{
 9336     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9337                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9338                /*weak*/ false, $res$$Register);
 9339   %}
 9340   ins_pipe(pipe_slow);
 9341 %}
 9342 
 9343 // This pattern is generated automatically from cas.m4.
 9344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9345 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9346 
 9347   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9348   ins_cost(2 * VOLATILE_REF_COST);
 9349   effect(TEMP_DEF res, KILL cr);
 9350   format %{
 9351     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9352   %}
 9353   ins_encode %{
 9354     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9355                Assembler::word, /*acquire*/ false, /*release*/ true,
 9356                /*weak*/ false, $res$$Register);
 9357   %}
 9358   ins_pipe(pipe_slow);
 9359 %}
 9360 
 9361 // This pattern is generated automatically from cas.m4.
 9362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9363 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9364   predicate(n->as_LoadStore()->barrier_data() == 0);
 9365   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9366   ins_cost(2 * VOLATILE_REF_COST);
 9367   effect(TEMP_DEF res, KILL cr);
 9368   format %{
 9369     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9370   %}
 9371   ins_encode %{
 9372     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9373                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9374                /*weak*/ false, $res$$Register);
 9375   %}
 9376   ins_pipe(pipe_slow);
 9377 %}
 9378 
 9379 // This pattern is generated automatically from cas.m4.
 9380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9381 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9382   predicate(needs_acquiring_load_exclusive(n));
 9383   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9384   ins_cost(VOLATILE_REF_COST);
 9385   effect(TEMP_DEF res, KILL cr);
 9386   format %{
 9387     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9388   %}
 9389   ins_encode %{
 9390     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9391                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9392                /*weak*/ false, $res$$Register);
 9393     __ sxtbw($res$$Register, $res$$Register);
 9394   %}
 9395   ins_pipe(pipe_slow);
 9396 %}
 9397 
 9398 // This pattern is generated automatically from cas.m4.
 9399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9400 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9401   predicate(needs_acquiring_load_exclusive(n));
 9402   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9403   ins_cost(VOLATILE_REF_COST);
 9404   effect(TEMP_DEF res, KILL cr);
 9405   format %{
 9406     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9407   %}
 9408   ins_encode %{
 9409     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9410                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9411                /*weak*/ false, $res$$Register);
 9412     __ sxthw($res$$Register, $res$$Register);
 9413   %}
 9414   ins_pipe(pipe_slow);
 9415 %}
 9416 
 9417 // This pattern is generated automatically from cas.m4.
 9418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9419 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9420   predicate(needs_acquiring_load_exclusive(n));
 9421   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9422   ins_cost(VOLATILE_REF_COST);
 9423   effect(TEMP_DEF res, KILL cr);
 9424   format %{
 9425     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9426   %}
 9427   ins_encode %{
 9428     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9429                Assembler::word, /*acquire*/ true, /*release*/ true,
 9430                /*weak*/ false, $res$$Register);
 9431   %}
 9432   ins_pipe(pipe_slow);
 9433 %}
 9434 
 9435 // This pattern is generated automatically from cas.m4.
 9436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9437 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9438   predicate(needs_acquiring_load_exclusive(n));
 9439   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9440   ins_cost(VOLATILE_REF_COST);
 9441   effect(TEMP_DEF res, KILL cr);
 9442   format %{
 9443     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9444   %}
 9445   ins_encode %{
 9446     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9447                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9448                /*weak*/ false, $res$$Register);
 9449   %}
 9450   ins_pipe(pipe_slow);
 9451 %}
 9452 
 9453 // This pattern is generated automatically from cas.m4.
 9454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9455 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9456   predicate(needs_acquiring_load_exclusive(n));
 9457   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9458   ins_cost(VOLATILE_REF_COST);
 9459   effect(TEMP_DEF res, KILL cr);
 9460   format %{
 9461     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9462   %}
 9463   ins_encode %{
 9464     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9465                Assembler::word, /*acquire*/ true, /*release*/ true,
 9466                /*weak*/ false, $res$$Register);
 9467   %}
 9468   ins_pipe(pipe_slow);
 9469 %}
 9470 
 9471 // This pattern is generated automatically from cas.m4.
 9472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9473 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9474   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9475   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9476   ins_cost(VOLATILE_REF_COST);
 9477   effect(TEMP_DEF res, KILL cr);
 9478   format %{
 9479     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9480   %}
 9481   ins_encode %{
 9482     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9483                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9484                /*weak*/ false, $res$$Register);
 9485   %}
 9486   ins_pipe(pipe_slow);
 9487 %}
 9488 
 9489 // This pattern is generated automatically from cas.m4.
 9490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9491 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9492 
 9493   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9494   ins_cost(2 * VOLATILE_REF_COST);
 9495   effect(KILL cr);
 9496   format %{
 9497     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9498     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9499   %}
 9500   ins_encode %{
 9501     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9502                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9503                /*weak*/ true, noreg);
 9504     __ csetw($res$$Register, Assembler::EQ);
 9505   %}
 9506   ins_pipe(pipe_slow);
 9507 %}
 9508 
 9509 // This pattern is generated automatically from cas.m4.
 9510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9511 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9512 
 9513   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9514   ins_cost(2 * VOLATILE_REF_COST);
 9515   effect(KILL cr);
 9516   format %{
 9517     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9518     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9519   %}
 9520   ins_encode %{
 9521     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9522                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9523                /*weak*/ true, noreg);
 9524     __ csetw($res$$Register, Assembler::EQ);
 9525   %}
 9526   ins_pipe(pipe_slow);
 9527 %}
 9528 
 9529 // This pattern is generated automatically from cas.m4.
 9530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9531 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9532 
 9533   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9534   ins_cost(2 * VOLATILE_REF_COST);
 9535   effect(KILL cr);
 9536   format %{
 9537     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9538     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9539   %}
 9540   ins_encode %{
 9541     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9542                Assembler::word, /*acquire*/ false, /*release*/ true,
 9543                /*weak*/ true, noreg);
 9544     __ csetw($res$$Register, Assembler::EQ);
 9545   %}
 9546   ins_pipe(pipe_slow);
 9547 %}
 9548 
 9549 // This pattern is generated automatically from cas.m4.
 9550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9551 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9552 
 9553   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9554   ins_cost(2 * VOLATILE_REF_COST);
 9555   effect(KILL cr);
 9556   format %{
 9557     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9558     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9559   %}
 9560   ins_encode %{
 9561     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9562                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9563                /*weak*/ true, noreg);
 9564     __ csetw($res$$Register, Assembler::EQ);
 9565   %}
 9566   ins_pipe(pipe_slow);
 9567 %}
 9568 
 9569 // This pattern is generated automatically from cas.m4.
 9570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9571 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9572 
 9573   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9574   ins_cost(2 * VOLATILE_REF_COST);
 9575   effect(KILL cr);
 9576   format %{
 9577     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9578     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9579   %}
 9580   ins_encode %{
 9581     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9582                Assembler::word, /*acquire*/ false, /*release*/ true,
 9583                /*weak*/ true, noreg);
 9584     __ csetw($res$$Register, Assembler::EQ);
 9585   %}
 9586   ins_pipe(pipe_slow);
 9587 %}
 9588 
 9589 // This pattern is generated automatically from cas.m4.
 9590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9591 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9592   predicate(n->as_LoadStore()->barrier_data() == 0);
 9593   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9594   ins_cost(2 * VOLATILE_REF_COST);
 9595   effect(KILL cr);
 9596   format %{
 9597     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9598     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9599   %}
 9600   ins_encode %{
 9601     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9602                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9603                /*weak*/ true, noreg);
 9604     __ csetw($res$$Register, Assembler::EQ);
 9605   %}
 9606   ins_pipe(pipe_slow);
 9607 %}
 9608 
 9609 // This pattern is generated automatically from cas.m4.
 9610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9611 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9612   predicate(needs_acquiring_load_exclusive(n));
 9613   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9614   ins_cost(VOLATILE_REF_COST);
 9615   effect(KILL cr);
 9616   format %{
 9617     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9618     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9619   %}
 9620   ins_encode %{
 9621     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9622                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9623                /*weak*/ true, noreg);
 9624     __ csetw($res$$Register, Assembler::EQ);
 9625   %}
 9626   ins_pipe(pipe_slow);
 9627 %}
 9628 
 9629 // This pattern is generated automatically from cas.m4.
 9630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9631 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9632   predicate(needs_acquiring_load_exclusive(n));
 9633   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9634   ins_cost(VOLATILE_REF_COST);
 9635   effect(KILL cr);
 9636   format %{
 9637     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9638     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9639   %}
 9640   ins_encode %{
 9641     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9642                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9643                /*weak*/ true, noreg);
 9644     __ csetw($res$$Register, Assembler::EQ);
 9645   %}
 9646   ins_pipe(pipe_slow);
 9647 %}
 9648 
 9649 // This pattern is generated automatically from cas.m4.
 9650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9651 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9652   predicate(needs_acquiring_load_exclusive(n));
 9653   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9654   ins_cost(VOLATILE_REF_COST);
 9655   effect(KILL cr);
 9656   format %{
 9657     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9658     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9659   %}
 9660   ins_encode %{
 9661     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9662                Assembler::word, /*acquire*/ true, /*release*/ true,
 9663                /*weak*/ true, noreg);
 9664     __ csetw($res$$Register, Assembler::EQ);
 9665   %}
 9666   ins_pipe(pipe_slow);
 9667 %}
 9668 
 9669 // This pattern is generated automatically from cas.m4.
 9670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9671 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9672   predicate(needs_acquiring_load_exclusive(n));
 9673   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9674   ins_cost(VOLATILE_REF_COST);
 9675   effect(KILL cr);
 9676   format %{
 9677     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9678     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9679   %}
 9680   ins_encode %{
 9681     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9682                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9683                /*weak*/ true, noreg);
 9684     __ csetw($res$$Register, Assembler::EQ);
 9685   %}
 9686   ins_pipe(pipe_slow);
 9687 %}
 9688 
 9689 // This pattern is generated automatically from cas.m4.
 9690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9691 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9692   predicate(needs_acquiring_load_exclusive(n));
 9693   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9694   ins_cost(VOLATILE_REF_COST);
 9695   effect(KILL cr);
 9696   format %{
 9697     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9698     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9699   %}
 9700   ins_encode %{
 9701     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9702                Assembler::word, /*acquire*/ true, /*release*/ true,
 9703                /*weak*/ true, noreg);
 9704     __ csetw($res$$Register, Assembler::EQ);
 9705   %}
 9706   ins_pipe(pipe_slow);
 9707 %}
 9708 
 9709 // This pattern is generated automatically from cas.m4.
 9710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9711 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9712   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9713   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9714   ins_cost(VOLATILE_REF_COST);
 9715   effect(KILL cr);
 9716   format %{
 9717     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9718     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9719   %}
 9720   ins_encode %{
 9721     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9722                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9723                /*weak*/ true, noreg);
 9724     __ csetw($res$$Register, Assembler::EQ);
 9725   %}
 9726   ins_pipe(pipe_slow);
 9727 %}
 9728 
 9729 // END This section of the file is automatically generated. Do not edit --------------
 9730 // ---------------------------------------------------------------------
 9731 
 9732 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9733   match(Set prev (GetAndSetI mem newv));
 9734   ins_cost(2 * VOLATILE_REF_COST);
 9735   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9736   ins_encode %{
 9737     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9738   %}
 9739   ins_pipe(pipe_serial);
 9740 %}
 9741 
 9742 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9743   match(Set prev (GetAndSetL mem newv));
 9744   ins_cost(2 * VOLATILE_REF_COST);
 9745   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9746   ins_encode %{
 9747     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9748   %}
 9749   ins_pipe(pipe_serial);
 9750 %}
 9751 
 9752 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9753   match(Set prev (GetAndSetN mem newv));
 9754   ins_cost(2 * VOLATILE_REF_COST);
 9755   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9756   ins_encode %{
 9757     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9758   %}
 9759   ins_pipe(pipe_serial);
 9760 %}
 9761 
 9762 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9763   predicate(n->as_LoadStore()->barrier_data() == 0);
 9764   match(Set prev (GetAndSetP mem newv));
 9765   ins_cost(2 * VOLATILE_REF_COST);
 9766   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9767   ins_encode %{
 9768     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9769   %}
 9770   ins_pipe(pipe_serial);
 9771 %}
 9772 
 9773 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9774   predicate(needs_acquiring_load_exclusive(n));
 9775   match(Set prev (GetAndSetI mem newv));
 9776   ins_cost(VOLATILE_REF_COST);
 9777   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9778   ins_encode %{
 9779     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9780   %}
 9781   ins_pipe(pipe_serial);
 9782 %}
 9783 
 9784 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9785   predicate(needs_acquiring_load_exclusive(n));
 9786   match(Set prev (GetAndSetL mem newv));
 9787   ins_cost(VOLATILE_REF_COST);
 9788   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9789   ins_encode %{
 9790     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9791   %}
 9792   ins_pipe(pipe_serial);
 9793 %}
 9794 
 9795 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9796   predicate(needs_acquiring_load_exclusive(n));
 9797   match(Set prev (GetAndSetN mem newv));
 9798   ins_cost(VOLATILE_REF_COST);
 9799   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9800   ins_encode %{
 9801     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9802   %}
 9803   ins_pipe(pipe_serial);
 9804 %}
 9805 
 9806 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9807   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9808   match(Set prev (GetAndSetP mem newv));
 9809   ins_cost(VOLATILE_REF_COST);
 9810   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9811   ins_encode %{
 9812     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9813   %}
 9814   ins_pipe(pipe_serial);
 9815 %}
 9816 
 9817 
 9818 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9819   match(Set newval (GetAndAddL mem incr));
 9820   ins_cost(2 * VOLATILE_REF_COST + 1);
 9821   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9822   ins_encode %{
 9823     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9824   %}
 9825   ins_pipe(pipe_serial);
 9826 %}
 9827 
 9828 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9829   predicate(n->as_LoadStore()->result_not_used());
 9830   match(Set dummy (GetAndAddL mem incr));
 9831   ins_cost(2 * VOLATILE_REF_COST);
 9832   format %{ "get_and_addL [$mem], $incr" %}
 9833   ins_encode %{
 9834     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9835   %}
 9836   ins_pipe(pipe_serial);
 9837 %}
 9838 
 9839 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9840   match(Set newval (GetAndAddL mem incr));
 9841   ins_cost(2 * VOLATILE_REF_COST + 1);
 9842   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9843   ins_encode %{
 9844     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9845   %}
 9846   ins_pipe(pipe_serial);
 9847 %}
 9848 
 9849 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9850   predicate(n->as_LoadStore()->result_not_used());
 9851   match(Set dummy (GetAndAddL mem incr));
 9852   ins_cost(2 * VOLATILE_REF_COST);
 9853   format %{ "get_and_addL [$mem], $incr" %}
 9854   ins_encode %{
 9855     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9856   %}
 9857   ins_pipe(pipe_serial);
 9858 %}
 9859 
 9860 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9861   match(Set newval (GetAndAddI mem incr));
 9862   ins_cost(2 * VOLATILE_REF_COST + 1);
 9863   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9864   ins_encode %{
 9865     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9866   %}
 9867   ins_pipe(pipe_serial);
 9868 %}
 9869 
 9870 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9871   predicate(n->as_LoadStore()->result_not_used());
 9872   match(Set dummy (GetAndAddI mem incr));
 9873   ins_cost(2 * VOLATILE_REF_COST);
 9874   format %{ "get_and_addI [$mem], $incr" %}
 9875   ins_encode %{
 9876     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9877   %}
 9878   ins_pipe(pipe_serial);
 9879 %}
 9880 
 9881 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9882   match(Set newval (GetAndAddI mem incr));
 9883   ins_cost(2 * VOLATILE_REF_COST + 1);
 9884   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9885   ins_encode %{
 9886     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9887   %}
 9888   ins_pipe(pipe_serial);
 9889 %}
 9890 
 9891 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9892   predicate(n->as_LoadStore()->result_not_used());
 9893   match(Set dummy (GetAndAddI mem incr));
 9894   ins_cost(2 * VOLATILE_REF_COST);
 9895   format %{ "get_and_addI [$mem], $incr" %}
 9896   ins_encode %{
 9897     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9898   %}
 9899   ins_pipe(pipe_serial);
 9900 %}
 9901 
 9902 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9903   predicate(needs_acquiring_load_exclusive(n));
 9904   match(Set newval (GetAndAddL mem incr));
 9905   ins_cost(VOLATILE_REF_COST + 1);
 9906   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9907   ins_encode %{
 9908     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9909   %}
 9910   ins_pipe(pipe_serial);
 9911 %}
 9912 
 9913 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9914   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9915   match(Set dummy (GetAndAddL mem incr));
 9916   ins_cost(VOLATILE_REF_COST);
 9917   format %{ "get_and_addL_acq [$mem], $incr" %}
 9918   ins_encode %{
 9919     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9920   %}
 9921   ins_pipe(pipe_serial);
 9922 %}
 9923 
 9924 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9925   predicate(needs_acquiring_load_exclusive(n));
 9926   match(Set newval (GetAndAddL mem incr));
 9927   ins_cost(VOLATILE_REF_COST + 1);
 9928   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9929   ins_encode %{
 9930     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9931   %}
 9932   ins_pipe(pipe_serial);
 9933 %}
 9934 
 9935 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9936   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9937   match(Set dummy (GetAndAddL mem incr));
 9938   ins_cost(VOLATILE_REF_COST);
 9939   format %{ "get_and_addL_acq [$mem], $incr" %}
 9940   ins_encode %{
 9941     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9942   %}
 9943   ins_pipe(pipe_serial);
 9944 %}
 9945 
 9946 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9947   predicate(needs_acquiring_load_exclusive(n));
 9948   match(Set newval (GetAndAddI mem incr));
 9949   ins_cost(VOLATILE_REF_COST + 1);
 9950   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9951   ins_encode %{
 9952     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9953   %}
 9954   ins_pipe(pipe_serial);
 9955 %}
 9956 
 9957 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9958   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9959   match(Set dummy (GetAndAddI mem incr));
 9960   ins_cost(VOLATILE_REF_COST);
 9961   format %{ "get_and_addI_acq [$mem], $incr" %}
 9962   ins_encode %{
 9963     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9964   %}
 9965   ins_pipe(pipe_serial);
 9966 %}
 9967 
 9968 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9969   predicate(needs_acquiring_load_exclusive(n));
 9970   match(Set newval (GetAndAddI mem incr));
 9971   ins_cost(VOLATILE_REF_COST + 1);
 9972   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9973   ins_encode %{
 9974     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9975   %}
 9976   ins_pipe(pipe_serial);
 9977 %}
 9978 
 9979 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9980   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9981   match(Set dummy (GetAndAddI mem incr));
 9982   ins_cost(VOLATILE_REF_COST);
 9983   format %{ "get_and_addI_acq [$mem], $incr" %}
 9984   ins_encode %{
 9985     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9986   %}
 9987   ins_pipe(pipe_serial);
 9988 %}
 9989 
 9990 // Manifest a CmpL result in an integer register.
 9991 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9992 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9993 %{
 9994   match(Set dst (CmpL3 src1 src2));
 9995   effect(KILL flags);
 9996 
 9997   ins_cost(INSN_COST * 6);
 9998   format %{
 9999       "cmp $src1, $src2"
10000       "csetw $dst, ne"
10001       "cnegw $dst, lt"
10002   %}
10003   // format %{ "CmpL3 $dst, $src1, $src2" %}
10004   ins_encode %{
10005     __ cmp($src1$$Register, $src2$$Register);
10006     __ csetw($dst$$Register, Assembler::NE);
10007     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10008   %}
10009 
10010   ins_pipe(pipe_class_default);
10011 %}
10012 
10013 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10014 %{
10015   match(Set dst (CmpL3 src1 src2));
10016   effect(KILL flags);
10017 
10018   ins_cost(INSN_COST * 6);
10019   format %{
10020       "cmp $src1, $src2"
10021       "csetw $dst, ne"
10022       "cnegw $dst, lt"
10023   %}
10024   ins_encode %{
10025     int32_t con = (int32_t)$src2$$constant;
10026      if (con < 0) {
10027       __ adds(zr, $src1$$Register, -con);
10028     } else {
10029       __ subs(zr, $src1$$Register, con);
10030     }
10031     __ csetw($dst$$Register, Assembler::NE);
10032     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10033   %}
10034 
10035   ins_pipe(pipe_class_default);
10036 %}
10037 
10038 // ============================================================================
10039 // Conditional Move Instructions
10040 
10041 // n.b. we have identical rules for both a signed compare op (cmpOp)
10042 // and an unsigned compare op (cmpOpU). it would be nice if we could
10043 // define an op class which merged both inputs and use it to type the
10044 // argument to a single rule. unfortunatelyt his fails because the
10045 // opclass does not live up to the COND_INTER interface of its
10046 // component operands. When the generic code tries to negate the
10047 // operand it ends up running the generci Machoper::negate method
10048 // which throws a ShouldNotHappen. So, we have to provide two flavours
10049 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10050 
10051 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10052   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10053 
10054   ins_cost(INSN_COST * 2);
10055   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10056 
10057   ins_encode %{
10058     __ cselw(as_Register($dst$$reg),
10059              as_Register($src2$$reg),
10060              as_Register($src1$$reg),
10061              (Assembler::Condition)$cmp$$cmpcode);
10062   %}
10063 
10064   ins_pipe(icond_reg_reg);
10065 %}
10066 
10067 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10068   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10069 
10070   ins_cost(INSN_COST * 2);
10071   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10072 
10073   ins_encode %{
10074     __ cselw(as_Register($dst$$reg),
10075              as_Register($src2$$reg),
10076              as_Register($src1$$reg),
10077              (Assembler::Condition)$cmp$$cmpcode);
10078   %}
10079 
10080   ins_pipe(icond_reg_reg);
10081 %}
10082 
10083 // special cases where one arg is zero
10084 
10085 // n.b. this is selected in preference to the rule above because it
10086 // avoids loading constant 0 into a source register
10087 
10088 // TODO
10089 // we ought only to be able to cull one of these variants as the ideal
10090 // transforms ought always to order the zero consistently (to left/right?)
10091 
10092 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10093   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10094 
10095   ins_cost(INSN_COST * 2);
10096   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10097 
10098   ins_encode %{
10099     __ cselw(as_Register($dst$$reg),
10100              as_Register($src$$reg),
10101              zr,
10102              (Assembler::Condition)$cmp$$cmpcode);
10103   %}
10104 
10105   ins_pipe(icond_reg);
10106 %}
10107 
10108 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10109   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10110 
10111   ins_cost(INSN_COST * 2);
10112   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10113 
10114   ins_encode %{
10115     __ cselw(as_Register($dst$$reg),
10116              as_Register($src$$reg),
10117              zr,
10118              (Assembler::Condition)$cmp$$cmpcode);
10119   %}
10120 
10121   ins_pipe(icond_reg);
10122 %}
10123 
10124 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10125   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10126 
10127   ins_cost(INSN_COST * 2);
10128   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10129 
10130   ins_encode %{
10131     __ cselw(as_Register($dst$$reg),
10132              zr,
10133              as_Register($src$$reg),
10134              (Assembler::Condition)$cmp$$cmpcode);
10135   %}
10136 
10137   ins_pipe(icond_reg);
10138 %}
10139 
10140 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10141   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10142 
10143   ins_cost(INSN_COST * 2);
10144   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10145 
10146   ins_encode %{
10147     __ cselw(as_Register($dst$$reg),
10148              zr,
10149              as_Register($src$$reg),
10150              (Assembler::Condition)$cmp$$cmpcode);
10151   %}
10152 
10153   ins_pipe(icond_reg);
10154 %}
10155 
10156 // special case for creating a boolean 0 or 1
10157 
10158 // n.b. this is selected in preference to the rule above because it
10159 // avoids loading constants 0 and 1 into a source register
10160 
10161 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10162   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10163 
10164   ins_cost(INSN_COST * 2);
10165   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10166 
10167   ins_encode %{
10168     // equivalently
10169     // cset(as_Register($dst$$reg),
10170     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10171     __ csincw(as_Register($dst$$reg),
10172              zr,
10173              zr,
10174              (Assembler::Condition)$cmp$$cmpcode);
10175   %}
10176 
10177   ins_pipe(icond_none);
10178 %}
10179 
10180 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10181   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10182 
10183   ins_cost(INSN_COST * 2);
10184   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10185 
10186   ins_encode %{
10187     // equivalently
10188     // cset(as_Register($dst$$reg),
10189     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10190     __ csincw(as_Register($dst$$reg),
10191              zr,
10192              zr,
10193              (Assembler::Condition)$cmp$$cmpcode);
10194   %}
10195 
10196   ins_pipe(icond_none);
10197 %}
10198 
10199 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10200   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10201 
10202   ins_cost(INSN_COST * 2);
10203   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10204 
10205   ins_encode %{
10206     __ csel(as_Register($dst$$reg),
10207             as_Register($src2$$reg),
10208             as_Register($src1$$reg),
10209             (Assembler::Condition)$cmp$$cmpcode);
10210   %}
10211 
10212   ins_pipe(icond_reg_reg);
10213 %}
10214 
10215 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10216   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10217 
10218   ins_cost(INSN_COST * 2);
10219   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10220 
10221   ins_encode %{
10222     __ csel(as_Register($dst$$reg),
10223             as_Register($src2$$reg),
10224             as_Register($src1$$reg),
10225             (Assembler::Condition)$cmp$$cmpcode);
10226   %}
10227 
10228   ins_pipe(icond_reg_reg);
10229 %}
10230 
10231 // special cases where one arg is zero
10232 
10233 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10234   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10235 
10236   ins_cost(INSN_COST * 2);
10237   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10238 
10239   ins_encode %{
10240     __ csel(as_Register($dst$$reg),
10241             zr,
10242             as_Register($src$$reg),
10243             (Assembler::Condition)$cmp$$cmpcode);
10244   %}
10245 
10246   ins_pipe(icond_reg);
10247 %}
10248 
10249 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10250   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10251 
10252   ins_cost(INSN_COST * 2);
10253   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10254 
10255   ins_encode %{
10256     __ csel(as_Register($dst$$reg),
10257             zr,
10258             as_Register($src$$reg),
10259             (Assembler::Condition)$cmp$$cmpcode);
10260   %}
10261 
10262   ins_pipe(icond_reg);
10263 %}
10264 
10265 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10266   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10267 
10268   ins_cost(INSN_COST * 2);
10269   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10270 
10271   ins_encode %{
10272     __ csel(as_Register($dst$$reg),
10273             as_Register($src$$reg),
10274             zr,
10275             (Assembler::Condition)$cmp$$cmpcode);
10276   %}
10277 
10278   ins_pipe(icond_reg);
10279 %}
10280 
10281 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10282   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10283 
10284   ins_cost(INSN_COST * 2);
10285   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10286 
10287   ins_encode %{
10288     __ csel(as_Register($dst$$reg),
10289             as_Register($src$$reg),
10290             zr,
10291             (Assembler::Condition)$cmp$$cmpcode);
10292   %}
10293 
10294   ins_pipe(icond_reg);
10295 %}
10296 
10297 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10298   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10299 
10300   ins_cost(INSN_COST * 2);
10301   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10302 
10303   ins_encode %{
10304     __ csel(as_Register($dst$$reg),
10305             as_Register($src2$$reg),
10306             as_Register($src1$$reg),
10307             (Assembler::Condition)$cmp$$cmpcode);
10308   %}
10309 
10310   ins_pipe(icond_reg_reg);
10311 %}
10312 
10313 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10314   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10315 
10316   ins_cost(INSN_COST * 2);
10317   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10318 
10319   ins_encode %{
10320     __ csel(as_Register($dst$$reg),
10321             as_Register($src2$$reg),
10322             as_Register($src1$$reg),
10323             (Assembler::Condition)$cmp$$cmpcode);
10324   %}
10325 
10326   ins_pipe(icond_reg_reg);
10327 %}
10328 
10329 // special cases where one arg is zero
10330 
10331 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10332   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10333 
10334   ins_cost(INSN_COST * 2);
10335   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10336 
10337   ins_encode %{
10338     __ csel(as_Register($dst$$reg),
10339             zr,
10340             as_Register($src$$reg),
10341             (Assembler::Condition)$cmp$$cmpcode);
10342   %}
10343 
10344   ins_pipe(icond_reg);
10345 %}
10346 
10347 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10348   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10349 
10350   ins_cost(INSN_COST * 2);
10351   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10352 
10353   ins_encode %{
10354     __ csel(as_Register($dst$$reg),
10355             zr,
10356             as_Register($src$$reg),
10357             (Assembler::Condition)$cmp$$cmpcode);
10358   %}
10359 
10360   ins_pipe(icond_reg);
10361 %}
10362 
10363 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10364   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10365 
10366   ins_cost(INSN_COST * 2);
10367   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10368 
10369   ins_encode %{
10370     __ csel(as_Register($dst$$reg),
10371             as_Register($src$$reg),
10372             zr,
10373             (Assembler::Condition)$cmp$$cmpcode);
10374   %}
10375 
10376   ins_pipe(icond_reg);
10377 %}
10378 
10379 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10380   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10381 
10382   ins_cost(INSN_COST * 2);
10383   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10384 
10385   ins_encode %{
10386     __ csel(as_Register($dst$$reg),
10387             as_Register($src$$reg),
10388             zr,
10389             (Assembler::Condition)$cmp$$cmpcode);
10390   %}
10391 
10392   ins_pipe(icond_reg);
10393 %}
10394 
10395 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10396   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10397 
10398   ins_cost(INSN_COST * 2);
10399   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10400 
10401   ins_encode %{
10402     __ cselw(as_Register($dst$$reg),
10403              as_Register($src2$$reg),
10404              as_Register($src1$$reg),
10405              (Assembler::Condition)$cmp$$cmpcode);
10406   %}
10407 
10408   ins_pipe(icond_reg_reg);
10409 %}
10410 
10411 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10412   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10413 
10414   ins_cost(INSN_COST * 2);
10415   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10416 
10417   ins_encode %{
10418     __ cselw(as_Register($dst$$reg),
10419              as_Register($src2$$reg),
10420              as_Register($src1$$reg),
10421              (Assembler::Condition)$cmp$$cmpcode);
10422   %}
10423 
10424   ins_pipe(icond_reg_reg);
10425 %}
10426 
10427 // special cases where one arg is zero
10428 
10429 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10430   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10431 
10432   ins_cost(INSN_COST * 2);
10433   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10434 
10435   ins_encode %{
10436     __ cselw(as_Register($dst$$reg),
10437              zr,
10438              as_Register($src$$reg),
10439              (Assembler::Condition)$cmp$$cmpcode);
10440   %}
10441 
10442   ins_pipe(icond_reg);
10443 %}
10444 
10445 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10446   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10447 
10448   ins_cost(INSN_COST * 2);
10449   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10450 
10451   ins_encode %{
10452     __ cselw(as_Register($dst$$reg),
10453              zr,
10454              as_Register($src$$reg),
10455              (Assembler::Condition)$cmp$$cmpcode);
10456   %}
10457 
10458   ins_pipe(icond_reg);
10459 %}
10460 
10461 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10462   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10463 
10464   ins_cost(INSN_COST * 2);
10465   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10466 
10467   ins_encode %{
10468     __ cselw(as_Register($dst$$reg),
10469              as_Register($src$$reg),
10470              zr,
10471              (Assembler::Condition)$cmp$$cmpcode);
10472   %}
10473 
10474   ins_pipe(icond_reg);
10475 %}
10476 
10477 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10478   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10479 
10480   ins_cost(INSN_COST * 2);
10481   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10482 
10483   ins_encode %{
10484     __ cselw(as_Register($dst$$reg),
10485              as_Register($src$$reg),
10486              zr,
10487              (Assembler::Condition)$cmp$$cmpcode);
10488   %}
10489 
10490   ins_pipe(icond_reg);
10491 %}
10492 
10493 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10494 %{
10495   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10496 
10497   ins_cost(INSN_COST * 3);
10498 
10499   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10500   ins_encode %{
10501     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10502     __ fcsels(as_FloatRegister($dst$$reg),
10503               as_FloatRegister($src2$$reg),
10504               as_FloatRegister($src1$$reg),
10505               cond);
10506   %}
10507 
10508   ins_pipe(fp_cond_reg_reg_s);
10509 %}
10510 
10511 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10512 %{
10513   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10514 
10515   ins_cost(INSN_COST * 3);
10516 
10517   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10518   ins_encode %{
10519     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10520     __ fcsels(as_FloatRegister($dst$$reg),
10521               as_FloatRegister($src2$$reg),
10522               as_FloatRegister($src1$$reg),
10523               cond);
10524   %}
10525 
10526   ins_pipe(fp_cond_reg_reg_s);
10527 %}
10528 
10529 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10530 %{
10531   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10532 
10533   ins_cost(INSN_COST * 3);
10534 
10535   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10536   ins_encode %{
10537     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10538     __ fcseld(as_FloatRegister($dst$$reg),
10539               as_FloatRegister($src2$$reg),
10540               as_FloatRegister($src1$$reg),
10541               cond);
10542   %}
10543 
10544   ins_pipe(fp_cond_reg_reg_d);
10545 %}
10546 
10547 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10548 %{
10549   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10550 
10551   ins_cost(INSN_COST * 3);
10552 
10553   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10554   ins_encode %{
10555     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10556     __ fcseld(as_FloatRegister($dst$$reg),
10557               as_FloatRegister($src2$$reg),
10558               as_FloatRegister($src1$$reg),
10559               cond);
10560   %}
10561 
10562   ins_pipe(fp_cond_reg_reg_d);
10563 %}
10564 
10565 // ============================================================================
10566 // Arithmetic Instructions
10567 //
10568 
10569 // Integer Addition
10570 
10571 // TODO
10572 // these currently employ operations which do not set CR and hence are
10573 // not flagged as killing CR but we would like to isolate the cases
10574 // where we want to set flags from those where we don't. need to work
10575 // out how to do that.
10576 
10577 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10578   match(Set dst (AddI src1 src2));
10579 
10580   ins_cost(INSN_COST);
10581   format %{ "addw  $dst, $src1, $src2" %}
10582 
10583   ins_encode %{
10584     __ addw(as_Register($dst$$reg),
10585             as_Register($src1$$reg),
10586             as_Register($src2$$reg));
10587   %}
10588 
10589   ins_pipe(ialu_reg_reg);
10590 %}
10591 
10592 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10593   match(Set dst (AddI src1 src2));
10594 
10595   ins_cost(INSN_COST);
10596   format %{ "addw $dst, $src1, $src2" %}
10597 
10598   // use opcode to indicate that this is an add not a sub
10599   opcode(0x0);
10600 
10601   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10602 
10603   ins_pipe(ialu_reg_imm);
10604 %}
10605 
10606 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10607   match(Set dst (AddI (ConvL2I src1) src2));
10608 
10609   ins_cost(INSN_COST);
10610   format %{ "addw $dst, $src1, $src2" %}
10611 
10612   // use opcode to indicate that this is an add not a sub
10613   opcode(0x0);
10614 
10615   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10616 
10617   ins_pipe(ialu_reg_imm);
10618 %}
10619 
10620 // Pointer Addition
10621 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10622   match(Set dst (AddP src1 src2));
10623 
10624   ins_cost(INSN_COST);
10625   format %{ "add $dst, $src1, $src2\t# ptr" %}
10626 
10627   ins_encode %{
10628     __ add(as_Register($dst$$reg),
10629            as_Register($src1$$reg),
10630            as_Register($src2$$reg));
10631   %}
10632 
10633   ins_pipe(ialu_reg_reg);
10634 %}
10635 
10636 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10637   match(Set dst (AddP src1 (ConvI2L src2)));
10638 
10639   ins_cost(1.9 * INSN_COST);
10640   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10641 
10642   ins_encode %{
10643     __ add(as_Register($dst$$reg),
10644            as_Register($src1$$reg),
10645            as_Register($src2$$reg), ext::sxtw);
10646   %}
10647 
10648   ins_pipe(ialu_reg_reg);
10649 %}
10650 
10651 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10652   match(Set dst (AddP src1 (LShiftL src2 scale)));
10653 
10654   ins_cost(1.9 * INSN_COST);
10655   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10656 
10657   ins_encode %{
10658     __ lea(as_Register($dst$$reg),
10659            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10660                    Address::lsl($scale$$constant)));
10661   %}
10662 
10663   ins_pipe(ialu_reg_reg_shift);
10664 %}
10665 
10666 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10667   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10668 
10669   ins_cost(1.9 * INSN_COST);
10670   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10671 
10672   ins_encode %{
10673     __ lea(as_Register($dst$$reg),
10674            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10675                    Address::sxtw($scale$$constant)));
10676   %}
10677 
10678   ins_pipe(ialu_reg_reg_shift);
10679 %}
10680 
10681 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10682   match(Set dst (LShiftL (ConvI2L src) scale));
10683 
10684   ins_cost(INSN_COST);
10685   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10686 
10687   ins_encode %{
10688     __ sbfiz(as_Register($dst$$reg),
10689           as_Register($src$$reg),
10690           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10691   %}
10692 
10693   ins_pipe(ialu_reg_shift);
10694 %}
10695 
10696 // Pointer Immediate Addition
10697 // n.b. this needs to be more expensive than using an indirect memory
10698 // operand
10699 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10700   match(Set dst (AddP src1 src2));
10701 
10702   ins_cost(INSN_COST);
10703   format %{ "add $dst, $src1, $src2\t# ptr" %}
10704 
10705   // use opcode to indicate that this is an add not a sub
10706   opcode(0x0);
10707 
10708   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10709 
10710   ins_pipe(ialu_reg_imm);
10711 %}
10712 
10713 // Long Addition
10714 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10715 
10716   match(Set dst (AddL src1 src2));
10717 
10718   ins_cost(INSN_COST);
10719   format %{ "add  $dst, $src1, $src2" %}
10720 
10721   ins_encode %{
10722     __ add(as_Register($dst$$reg),
10723            as_Register($src1$$reg),
10724            as_Register($src2$$reg));
10725   %}
10726 
10727   ins_pipe(ialu_reg_reg);
10728 %}
10729 
10730 // No constant pool entries requiredLong Immediate Addition.
10731 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10732   match(Set dst (AddL src1 src2));
10733 
10734   ins_cost(INSN_COST);
10735   format %{ "add $dst, $src1, $src2" %}
10736 
10737   // use opcode to indicate that this is an add not a sub
10738   opcode(0x0);
10739 
10740   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10741 
10742   ins_pipe(ialu_reg_imm);
10743 %}
10744 
10745 // Integer Subtraction
10746 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10747   match(Set dst (SubI src1 src2));
10748 
10749   ins_cost(INSN_COST);
10750   format %{ "subw  $dst, $src1, $src2" %}
10751 
10752   ins_encode %{
10753     __ subw(as_Register($dst$$reg),
10754             as_Register($src1$$reg),
10755             as_Register($src2$$reg));
10756   %}
10757 
10758   ins_pipe(ialu_reg_reg);
10759 %}
10760 
10761 // Immediate Subtraction
10762 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10763   match(Set dst (SubI src1 src2));
10764 
10765   ins_cost(INSN_COST);
10766   format %{ "subw $dst, $src1, $src2" %}
10767 
10768   // use opcode to indicate that this is a sub not an add
10769   opcode(0x1);
10770 
10771   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10772 
10773   ins_pipe(ialu_reg_imm);
10774 %}
10775 
10776 // Long Subtraction
10777 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10778 
10779   match(Set dst (SubL src1 src2));
10780 
10781   ins_cost(INSN_COST);
10782   format %{ "sub  $dst, $src1, $src2" %}
10783 
10784   ins_encode %{
10785     __ sub(as_Register($dst$$reg),
10786            as_Register($src1$$reg),
10787            as_Register($src2$$reg));
10788   %}
10789 
10790   ins_pipe(ialu_reg_reg);
10791 %}
10792 
10793 // No constant pool entries requiredLong Immediate Subtraction.
10794 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10795   match(Set dst (SubL src1 src2));
10796 
10797   ins_cost(INSN_COST);
10798   format %{ "sub$dst, $src1, $src2" %}
10799 
10800   // use opcode to indicate that this is a sub not an add
10801   opcode(0x1);
10802 
10803   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10804 
10805   ins_pipe(ialu_reg_imm);
10806 %}
10807 
10808 // Integer Negation (special case for sub)
10809 
10810 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10811   match(Set dst (SubI zero src));
10812 
10813   ins_cost(INSN_COST);
10814   format %{ "negw $dst, $src\t# int" %}
10815 
10816   ins_encode %{
10817     __ negw(as_Register($dst$$reg),
10818             as_Register($src$$reg));
10819   %}
10820 
10821   ins_pipe(ialu_reg);
10822 %}
10823 
10824 // Long Negation
10825 
10826 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10827   match(Set dst (SubL zero src));
10828 
10829   ins_cost(INSN_COST);
10830   format %{ "neg $dst, $src\t# long" %}
10831 
10832   ins_encode %{
10833     __ neg(as_Register($dst$$reg),
10834            as_Register($src$$reg));
10835   %}
10836 
10837   ins_pipe(ialu_reg);
10838 %}
10839 
10840 // Integer Multiply
10841 
10842 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10843   match(Set dst (MulI src1 src2));
10844 
10845   ins_cost(INSN_COST * 3);
10846   format %{ "mulw  $dst, $src1, $src2" %}
10847 
10848   ins_encode %{
10849     __ mulw(as_Register($dst$$reg),
10850             as_Register($src1$$reg),
10851             as_Register($src2$$reg));
10852   %}
10853 
10854   ins_pipe(imul_reg_reg);
10855 %}
10856 
10857 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10858   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10859 
10860   ins_cost(INSN_COST * 3);
10861   format %{ "smull  $dst, $src1, $src2" %}
10862 
10863   ins_encode %{
10864     __ smull(as_Register($dst$$reg),
10865              as_Register($src1$$reg),
10866              as_Register($src2$$reg));
10867   %}
10868 
10869   ins_pipe(imul_reg_reg);
10870 %}
10871 
10872 // Long Multiply
10873 
10874 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10875   match(Set dst (MulL src1 src2));
10876 
10877   ins_cost(INSN_COST * 5);
10878   format %{ "mul  $dst, $src1, $src2" %}
10879 
10880   ins_encode %{
10881     __ mul(as_Register($dst$$reg),
10882            as_Register($src1$$reg),
10883            as_Register($src2$$reg));
10884   %}
10885 
10886   ins_pipe(lmul_reg_reg);
10887 %}
10888 
10889 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10890 %{
10891   match(Set dst (MulHiL src1 src2));
10892 
10893   ins_cost(INSN_COST * 7);
10894   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10895 
10896   ins_encode %{
10897     __ smulh(as_Register($dst$$reg),
10898              as_Register($src1$$reg),
10899              as_Register($src2$$reg));
10900   %}
10901 
10902   ins_pipe(lmul_reg_reg);
10903 %}
10904 
10905 // Combined Integer Multiply & Add/Sub
10906 
10907 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10908   match(Set dst (AddI src3 (MulI src1 src2)));
10909 
10910   ins_cost(INSN_COST * 3);
10911   format %{ "madd  $dst, $src1, $src2, $src3" %}
10912 
10913   ins_encode %{
10914     __ maddw(as_Register($dst$$reg),
10915              as_Register($src1$$reg),
10916              as_Register($src2$$reg),
10917              as_Register($src3$$reg));
10918   %}
10919 
10920   ins_pipe(imac_reg_reg);
10921 %}
10922 
10923 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10924   match(Set dst (SubI src3 (MulI src1 src2)));
10925 
10926   ins_cost(INSN_COST * 3);
10927   format %{ "msub  $dst, $src1, $src2, $src3" %}
10928 
10929   ins_encode %{
10930     __ msubw(as_Register($dst$$reg),
10931              as_Register($src1$$reg),
10932              as_Register($src2$$reg),
10933              as_Register($src3$$reg));
10934   %}
10935 
10936   ins_pipe(imac_reg_reg);
10937 %}
10938 
10939 // Combined Integer Multiply & Neg
10940 
10941 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10942   match(Set dst (MulI (SubI zero src1) src2));
10943   match(Set dst (MulI src1 (SubI zero src2)));
10944 
10945   ins_cost(INSN_COST * 3);
10946   format %{ "mneg  $dst, $src1, $src2" %}
10947 
10948   ins_encode %{
10949     __ mnegw(as_Register($dst$$reg),
10950              as_Register($src1$$reg),
10951              as_Register($src2$$reg));
10952   %}
10953 
10954   ins_pipe(imac_reg_reg);
10955 %}
10956 
10957 // Combined Long Multiply & Add/Sub
10958 
10959 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10960   match(Set dst (AddL src3 (MulL src1 src2)));
10961 
10962   ins_cost(INSN_COST * 5);
10963   format %{ "madd  $dst, $src1, $src2, $src3" %}
10964 
10965   ins_encode %{
10966     __ madd(as_Register($dst$$reg),
10967             as_Register($src1$$reg),
10968             as_Register($src2$$reg),
10969             as_Register($src3$$reg));
10970   %}
10971 
10972   ins_pipe(lmac_reg_reg);
10973 %}
10974 
10975 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10976   match(Set dst (SubL src3 (MulL src1 src2)));
10977 
10978   ins_cost(INSN_COST * 5);
10979   format %{ "msub  $dst, $src1, $src2, $src3" %}
10980 
10981   ins_encode %{
10982     __ msub(as_Register($dst$$reg),
10983             as_Register($src1$$reg),
10984             as_Register($src2$$reg),
10985             as_Register($src3$$reg));
10986   %}
10987 
10988   ins_pipe(lmac_reg_reg);
10989 %}
10990 
10991 // Combined Long Multiply & Neg
10992 
10993 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10994   match(Set dst (MulL (SubL zero src1) src2));
10995   match(Set dst (MulL src1 (SubL zero src2)));
10996 
10997   ins_cost(INSN_COST * 5);
10998   format %{ "mneg  $dst, $src1, $src2" %}
10999 
11000   ins_encode %{
11001     __ mneg(as_Register($dst$$reg),
11002             as_Register($src1$$reg),
11003             as_Register($src2$$reg));
11004   %}
11005 
11006   ins_pipe(lmac_reg_reg);
11007 %}
11008 
11009 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11010 
11011 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11012   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11013 
11014   ins_cost(INSN_COST * 3);
11015   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11016 
11017   ins_encode %{
11018     __ smaddl(as_Register($dst$$reg),
11019               as_Register($src1$$reg),
11020               as_Register($src2$$reg),
11021               as_Register($src3$$reg));
11022   %}
11023 
11024   ins_pipe(imac_reg_reg);
11025 %}
11026 
11027 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11028   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11029 
11030   ins_cost(INSN_COST * 3);
11031   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11032 
11033   ins_encode %{
11034     __ smsubl(as_Register($dst$$reg),
11035               as_Register($src1$$reg),
11036               as_Register($src2$$reg),
11037               as_Register($src3$$reg));
11038   %}
11039 
11040   ins_pipe(imac_reg_reg);
11041 %}
11042 
11043 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11044   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11045   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11046 
11047   ins_cost(INSN_COST * 3);
11048   format %{ "smnegl  $dst, $src1, $src2" %}
11049 
11050   ins_encode %{
11051     __ smnegl(as_Register($dst$$reg),
11052               as_Register($src1$$reg),
11053               as_Register($src2$$reg));
11054   %}
11055 
11056   ins_pipe(imac_reg_reg);
11057 %}
11058 
11059 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11060 
11061 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11062   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11063 
11064   ins_cost(INSN_COST * 5);
11065   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11066             "maddw $dst, $src3, $src4, rscratch1" %}
11067 
11068   ins_encode %{
11069     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11070     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11071 
11072   ins_pipe(imac_reg_reg);
11073 %}
11074 
11075 // Integer Divide
11076 
11077 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11078   match(Set dst (DivI src1 src2));
11079 
11080   ins_cost(INSN_COST * 19);
11081   format %{ "sdivw  $dst, $src1, $src2" %}
11082 
11083   ins_encode(aarch64_enc_divw(dst, src1, src2));
11084   ins_pipe(idiv_reg_reg);
11085 %}
11086 
11087 // Long Divide
11088 
11089 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11090   match(Set dst (DivL src1 src2));
11091 
11092   ins_cost(INSN_COST * 35);
11093   format %{ "sdiv   $dst, $src1, $src2" %}
11094 
11095   ins_encode(aarch64_enc_div(dst, src1, src2));
11096   ins_pipe(ldiv_reg_reg);
11097 %}
11098 
11099 // Integer Remainder
11100 
11101 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11102   match(Set dst (ModI src1 src2));
11103 
11104   ins_cost(INSN_COST * 22);
11105   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11106             "msubw($dst, rscratch1, $src2, $src1" %}
11107 
11108   ins_encode(aarch64_enc_modw(dst, src1, src2));
11109   ins_pipe(idiv_reg_reg);
11110 %}
11111 
11112 // Long Remainder
11113 
11114 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11115   match(Set dst (ModL src1 src2));
11116 
11117   ins_cost(INSN_COST * 38);
11118   format %{ "sdiv   rscratch1, $src1, $src2\n"
11119             "msub($dst, rscratch1, $src2, $src1" %}
11120 
11121   ins_encode(aarch64_enc_mod(dst, src1, src2));
11122   ins_pipe(ldiv_reg_reg);
11123 %}
11124 
11125 // Integer Shifts
11126 
11127 // Shift Left Register
11128 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11129   match(Set dst (LShiftI src1 src2));
11130 
11131   ins_cost(INSN_COST * 2);
11132   format %{ "lslvw  $dst, $src1, $src2" %}
11133 
11134   ins_encode %{
11135     __ lslvw(as_Register($dst$$reg),
11136              as_Register($src1$$reg),
11137              as_Register($src2$$reg));
11138   %}
11139 
11140   ins_pipe(ialu_reg_reg_vshift);
11141 %}
11142 
11143 // Shift Left Immediate
11144 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11145   match(Set dst (LShiftI src1 src2));
11146 
11147   ins_cost(INSN_COST);
11148   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11149 
11150   ins_encode %{
11151     __ lslw(as_Register($dst$$reg),
11152             as_Register($src1$$reg),
11153             $src2$$constant & 0x1f);
11154   %}
11155 
11156   ins_pipe(ialu_reg_shift);
11157 %}
11158 
11159 // Shift Right Logical Register
11160 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11161   match(Set dst (URShiftI src1 src2));
11162 
11163   ins_cost(INSN_COST * 2);
11164   format %{ "lsrvw  $dst, $src1, $src2" %}
11165 
11166   ins_encode %{
11167     __ lsrvw(as_Register($dst$$reg),
11168              as_Register($src1$$reg),
11169              as_Register($src2$$reg));
11170   %}
11171 
11172   ins_pipe(ialu_reg_reg_vshift);
11173 %}
11174 
11175 // Shift Right Logical Immediate
11176 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11177   match(Set dst (URShiftI src1 src2));
11178 
11179   ins_cost(INSN_COST);
11180   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11181 
11182   ins_encode %{
11183     __ lsrw(as_Register($dst$$reg),
11184             as_Register($src1$$reg),
11185             $src2$$constant & 0x1f);
11186   %}
11187 
11188   ins_pipe(ialu_reg_shift);
11189 %}
11190 
11191 // Shift Right Arithmetic Register
11192 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11193   match(Set dst (RShiftI src1 src2));
11194 
11195   ins_cost(INSN_COST * 2);
11196   format %{ "asrvw  $dst, $src1, $src2" %}
11197 
11198   ins_encode %{
11199     __ asrvw(as_Register($dst$$reg),
11200              as_Register($src1$$reg),
11201              as_Register($src2$$reg));
11202   %}
11203 
11204   ins_pipe(ialu_reg_reg_vshift);
11205 %}
11206 
11207 // Shift Right Arithmetic Immediate
11208 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11209   match(Set dst (RShiftI src1 src2));
11210 
11211   ins_cost(INSN_COST);
11212   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11213 
11214   ins_encode %{
11215     __ asrw(as_Register($dst$$reg),
11216             as_Register($src1$$reg),
11217             $src2$$constant & 0x1f);
11218   %}
11219 
11220   ins_pipe(ialu_reg_shift);
11221 %}
11222 
11223 // Combined Int Mask and Right Shift (using UBFM)
11224 // TODO
11225 
11226 // Long Shifts
11227 
11228 // Shift Left Register
11229 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11230   match(Set dst (LShiftL src1 src2));
11231 
11232   ins_cost(INSN_COST * 2);
11233   format %{ "lslv  $dst, $src1, $src2" %}
11234 
11235   ins_encode %{
11236     __ lslv(as_Register($dst$$reg),
11237             as_Register($src1$$reg),
11238             as_Register($src2$$reg));
11239   %}
11240 
11241   ins_pipe(ialu_reg_reg_vshift);
11242 %}
11243 
11244 // Shift Left Immediate
11245 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11246   match(Set dst (LShiftL src1 src2));
11247 
11248   ins_cost(INSN_COST);
11249   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11250 
11251   ins_encode %{
11252     __ lsl(as_Register($dst$$reg),
11253             as_Register($src1$$reg),
11254             $src2$$constant & 0x3f);
11255   %}
11256 
11257   ins_pipe(ialu_reg_shift);
11258 %}
11259 
11260 // Shift Right Logical Register
11261 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11262   match(Set dst (URShiftL src1 src2));
11263 
11264   ins_cost(INSN_COST * 2);
11265   format %{ "lsrv  $dst, $src1, $src2" %}
11266 
11267   ins_encode %{
11268     __ lsrv(as_Register($dst$$reg),
11269             as_Register($src1$$reg),
11270             as_Register($src2$$reg));
11271   %}
11272 
11273   ins_pipe(ialu_reg_reg_vshift);
11274 %}
11275 
11276 // Shift Right Logical Immediate
11277 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11278   match(Set dst (URShiftL src1 src2));
11279 
11280   ins_cost(INSN_COST);
11281   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11282 
11283   ins_encode %{
11284     __ lsr(as_Register($dst$$reg),
11285            as_Register($src1$$reg),
11286            $src2$$constant & 0x3f);
11287   %}
11288 
11289   ins_pipe(ialu_reg_shift);
11290 %}
11291 
11292 // A special-case pattern for card table stores.
11293 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11294   match(Set dst (URShiftL (CastP2X src1) src2));
11295 
11296   ins_cost(INSN_COST);
11297   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11298 
11299   ins_encode %{
11300     __ lsr(as_Register($dst$$reg),
11301            as_Register($src1$$reg),
11302            $src2$$constant & 0x3f);
11303   %}
11304 
11305   ins_pipe(ialu_reg_shift);
11306 %}
11307 
11308 // Shift Right Arithmetic Register
11309 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11310   match(Set dst (RShiftL src1 src2));
11311 
11312   ins_cost(INSN_COST * 2);
11313   format %{ "asrv  $dst, $src1, $src2" %}
11314 
11315   ins_encode %{
11316     __ asrv(as_Register($dst$$reg),
11317             as_Register($src1$$reg),
11318             as_Register($src2$$reg));
11319   %}
11320 
11321   ins_pipe(ialu_reg_reg_vshift);
11322 %}
11323 
11324 // Shift Right Arithmetic Immediate
11325 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11326   match(Set dst (RShiftL src1 src2));
11327 
11328   ins_cost(INSN_COST);
11329   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11330 
11331   ins_encode %{
11332     __ asr(as_Register($dst$$reg),
11333            as_Register($src1$$reg),
11334            $src2$$constant & 0x3f);
11335   %}
11336 
11337   ins_pipe(ialu_reg_shift);
11338 %}
11339 
11340 // BEGIN This section of the file is automatically generated. Do not edit --------------
11341 // This section is generated from aarch64_ad.m4
11342 
11343 
11344 // This pattern is automatically generated from aarch64_ad.m4
11345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11346 instruct regL_not_reg(iRegLNoSp dst,
11347                          iRegL src1, immL_M1 m1,
11348                          rFlagsReg cr) %{
11349   match(Set dst (XorL src1 m1));
11350   ins_cost(INSN_COST);
11351   format %{ "eon  $dst, $src1, zr" %}
11352 
11353   ins_encode %{
11354     __ eon(as_Register($dst$$reg),
11355               as_Register($src1$$reg),
11356               zr,
11357               Assembler::LSL, 0);
11358   %}
11359 
11360   ins_pipe(ialu_reg);
11361 %}
11362 
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 instruct regI_not_reg(iRegINoSp dst,
11366                          iRegIorL2I src1, immI_M1 m1,
11367                          rFlagsReg cr) %{
11368   match(Set dst (XorI src1 m1));
11369   ins_cost(INSN_COST);
11370   format %{ "eonw  $dst, $src1, zr" %}
11371 
11372   ins_encode %{
11373     __ eonw(as_Register($dst$$reg),
11374               as_Register($src1$$reg),
11375               zr,
11376               Assembler::LSL, 0);
11377   %}
11378 
11379   ins_pipe(ialu_reg);
11380 %}
11381 
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 instruct AndI_reg_not_reg(iRegINoSp dst,
11385                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11386   match(Set dst (AndI src1 (XorI src2 m1)));
11387   ins_cost(INSN_COST);
11388   format %{ "bicw  $dst, $src1, $src2" %}
11389 
11390   ins_encode %{
11391     __ bicw(as_Register($dst$$reg),
11392               as_Register($src1$$reg),
11393               as_Register($src2$$reg),
11394               Assembler::LSL, 0);
11395   %}
11396 
11397   ins_pipe(ialu_reg_reg);
11398 %}
11399 
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 instruct AndL_reg_not_reg(iRegLNoSp dst,
11403                          iRegL src1, iRegL src2, immL_M1 m1) %{
11404   match(Set dst (AndL src1 (XorL src2 m1)));
11405   ins_cost(INSN_COST);
11406   format %{ "bic  $dst, $src1, $src2" %}
11407 
11408   ins_encode %{
11409     __ bic(as_Register($dst$$reg),
11410               as_Register($src1$$reg),
11411               as_Register($src2$$reg),
11412               Assembler::LSL, 0);
11413   %}
11414 
11415   ins_pipe(ialu_reg_reg);
11416 %}
11417 
11418 // This pattern is automatically generated from aarch64_ad.m4
11419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11420 instruct OrI_reg_not_reg(iRegINoSp dst,
11421                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11422   match(Set dst (OrI src1 (XorI src2 m1)));
11423   ins_cost(INSN_COST);
11424   format %{ "ornw  $dst, $src1, $src2" %}
11425 
11426   ins_encode %{
11427     __ ornw(as_Register($dst$$reg),
11428               as_Register($src1$$reg),
11429               as_Register($src2$$reg),
11430               Assembler::LSL, 0);
11431   %}
11432 
11433   ins_pipe(ialu_reg_reg);
11434 %}
11435 
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 instruct OrL_reg_not_reg(iRegLNoSp dst,
11439                          iRegL src1, iRegL src2, immL_M1 m1) %{
11440   match(Set dst (OrL src1 (XorL src2 m1)));
11441   ins_cost(INSN_COST);
11442   format %{ "orn  $dst, $src1, $src2" %}
11443 
11444   ins_encode %{
11445     __ orn(as_Register($dst$$reg),
11446               as_Register($src1$$reg),
11447               as_Register($src2$$reg),
11448               Assembler::LSL, 0);
11449   %}
11450 
11451   ins_pipe(ialu_reg_reg);
11452 %}
11453 
11454 // This pattern is automatically generated from aarch64_ad.m4
11455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11456 instruct XorI_reg_not_reg(iRegINoSp dst,
11457                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11458   match(Set dst (XorI m1 (XorI src2 src1)));
11459   ins_cost(INSN_COST);
11460   format %{ "eonw  $dst, $src1, $src2" %}
11461 
11462   ins_encode %{
11463     __ eonw(as_Register($dst$$reg),
11464               as_Register($src1$$reg),
11465               as_Register($src2$$reg),
11466               Assembler::LSL, 0);
11467   %}
11468 
11469   ins_pipe(ialu_reg_reg);
11470 %}
11471 
11472 // This pattern is automatically generated from aarch64_ad.m4
11473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11474 instruct XorL_reg_not_reg(iRegLNoSp dst,
11475                          iRegL src1, iRegL src2, immL_M1 m1) %{
11476   match(Set dst (XorL m1 (XorL src2 src1)));
11477   ins_cost(INSN_COST);
11478   format %{ "eon  $dst, $src1, $src2" %}
11479 
11480   ins_encode %{
11481     __ eon(as_Register($dst$$reg),
11482               as_Register($src1$$reg),
11483               as_Register($src2$$reg),
11484               Assembler::LSL, 0);
11485   %}
11486 
11487   ins_pipe(ialu_reg_reg);
11488 %}
11489 
11490 // This pattern is automatically generated from aarch64_ad.m4
11491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11492 // val & (-1 ^ (val >>> shift)) ==> bicw
11493 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11494                          iRegIorL2I src1, iRegIorL2I src2,
11495                          immI src3, immI_M1 src4) %{
11496   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11497   ins_cost(1.9 * INSN_COST);
11498   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11499 
11500   ins_encode %{
11501     __ bicw(as_Register($dst$$reg),
11502               as_Register($src1$$reg),
11503               as_Register($src2$$reg),
11504               Assembler::LSR,
11505               $src3$$constant & 0x1f);
11506   %}
11507 
11508   ins_pipe(ialu_reg_reg_shift);
11509 %}
11510 
11511 // This pattern is automatically generated from aarch64_ad.m4
11512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11513 // val & (-1 ^ (val >>> shift)) ==> bic
11514 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11515                          iRegL src1, iRegL src2,
11516                          immI src3, immL_M1 src4) %{
11517   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11518   ins_cost(1.9 * INSN_COST);
11519   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11520 
11521   ins_encode %{
11522     __ bic(as_Register($dst$$reg),
11523               as_Register($src1$$reg),
11524               as_Register($src2$$reg),
11525               Assembler::LSR,
11526               $src3$$constant & 0x3f);
11527   %}
11528 
11529   ins_pipe(ialu_reg_reg_shift);
11530 %}
11531 
11532 // This pattern is automatically generated from aarch64_ad.m4
11533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11534 // val & (-1 ^ (val >> shift)) ==> bicw
11535 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11536                          iRegIorL2I src1, iRegIorL2I src2,
11537                          immI src3, immI_M1 src4) %{
11538   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11539   ins_cost(1.9 * INSN_COST);
11540   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11541 
11542   ins_encode %{
11543     __ bicw(as_Register($dst$$reg),
11544               as_Register($src1$$reg),
11545               as_Register($src2$$reg),
11546               Assembler::ASR,
11547               $src3$$constant & 0x1f);
11548   %}
11549 
11550   ins_pipe(ialu_reg_reg_shift);
11551 %}
11552 
11553 // This pattern is automatically generated from aarch64_ad.m4
11554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11555 // val & (-1 ^ (val >> shift)) ==> bic
11556 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11557                          iRegL src1, iRegL src2,
11558                          immI src3, immL_M1 src4) %{
11559   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11560   ins_cost(1.9 * INSN_COST);
11561   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11562 
11563   ins_encode %{
11564     __ bic(as_Register($dst$$reg),
11565               as_Register($src1$$reg),
11566               as_Register($src2$$reg),
11567               Assembler::ASR,
11568               $src3$$constant & 0x3f);
11569   %}
11570 
11571   ins_pipe(ialu_reg_reg_shift);
11572 %}
11573 
11574 // This pattern is automatically generated from aarch64_ad.m4
11575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11576 // val & (-1 ^ (val ror shift)) ==> bicw
11577 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11578                          iRegIorL2I src1, iRegIorL2I src2,
11579                          immI src3, immI_M1 src4) %{
11580   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11581   ins_cost(1.9 * INSN_COST);
11582   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11583 
11584   ins_encode %{
11585     __ bicw(as_Register($dst$$reg),
11586               as_Register($src1$$reg),
11587               as_Register($src2$$reg),
11588               Assembler::ROR,
11589               $src3$$constant & 0x1f);
11590   %}
11591 
11592   ins_pipe(ialu_reg_reg_shift);
11593 %}
11594 
11595 // This pattern is automatically generated from aarch64_ad.m4
11596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11597 // val & (-1 ^ (val ror shift)) ==> bic
11598 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11599                          iRegL src1, iRegL src2,
11600                          immI src3, immL_M1 src4) %{
11601   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11602   ins_cost(1.9 * INSN_COST);
11603   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11604 
11605   ins_encode %{
11606     __ bic(as_Register($dst$$reg),
11607               as_Register($src1$$reg),
11608               as_Register($src2$$reg),
11609               Assembler::ROR,
11610               $src3$$constant & 0x3f);
11611   %}
11612 
11613   ins_pipe(ialu_reg_reg_shift);
11614 %}
11615 
11616 // This pattern is automatically generated from aarch64_ad.m4
11617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11618 // val & (-1 ^ (val << shift)) ==> bicw
11619 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11620                          iRegIorL2I src1, iRegIorL2I src2,
11621                          immI src3, immI_M1 src4) %{
11622   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11623   ins_cost(1.9 * INSN_COST);
11624   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11625 
11626   ins_encode %{
11627     __ bicw(as_Register($dst$$reg),
11628               as_Register($src1$$reg),
11629               as_Register($src2$$reg),
11630               Assembler::LSL,
11631               $src3$$constant & 0x1f);
11632   %}
11633 
11634   ins_pipe(ialu_reg_reg_shift);
11635 %}
11636 
11637 // This pattern is automatically generated from aarch64_ad.m4
11638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11639 // val & (-1 ^ (val << shift)) ==> bic
11640 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11641                          iRegL src1, iRegL src2,
11642                          immI src3, immL_M1 src4) %{
11643   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11644   ins_cost(1.9 * INSN_COST);
11645   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11646 
11647   ins_encode %{
11648     __ bic(as_Register($dst$$reg),
11649               as_Register($src1$$reg),
11650               as_Register($src2$$reg),
11651               Assembler::LSL,
11652               $src3$$constant & 0x3f);
11653   %}
11654 
11655   ins_pipe(ialu_reg_reg_shift);
11656 %}
11657 
11658 // This pattern is automatically generated from aarch64_ad.m4
11659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11660 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11661 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11662                          iRegIorL2I src1, iRegIorL2I src2,
11663                          immI src3, immI_M1 src4) %{
11664   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11665   ins_cost(1.9 * INSN_COST);
11666   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11667 
11668   ins_encode %{
11669     __ eonw(as_Register($dst$$reg),
11670               as_Register($src1$$reg),
11671               as_Register($src2$$reg),
11672               Assembler::LSR,
11673               $src3$$constant & 0x1f);
11674   %}
11675 
11676   ins_pipe(ialu_reg_reg_shift);
11677 %}
11678 
11679 // This pattern is automatically generated from aarch64_ad.m4
11680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11681 // val ^ (-1 ^ (val >>> shift)) ==> eon
11682 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11683                          iRegL src1, iRegL src2,
11684                          immI src3, immL_M1 src4) %{
11685   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11686   ins_cost(1.9 * INSN_COST);
11687   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11688 
11689   ins_encode %{
11690     __ eon(as_Register($dst$$reg),
11691               as_Register($src1$$reg),
11692               as_Register($src2$$reg),
11693               Assembler::LSR,
11694               $src3$$constant & 0x3f);
11695   %}
11696 
11697   ins_pipe(ialu_reg_reg_shift);
11698 %}
11699 
11700 // This pattern is automatically generated from aarch64_ad.m4
11701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11702 // val ^ (-1 ^ (val >> shift)) ==> eonw
11703 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11704                          iRegIorL2I src1, iRegIorL2I src2,
11705                          immI src3, immI_M1 src4) %{
11706   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11707   ins_cost(1.9 * INSN_COST);
11708   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11709 
11710   ins_encode %{
11711     __ eonw(as_Register($dst$$reg),
11712               as_Register($src1$$reg),
11713               as_Register($src2$$reg),
11714               Assembler::ASR,
11715               $src3$$constant & 0x1f);
11716   %}
11717 
11718   ins_pipe(ialu_reg_reg_shift);
11719 %}
11720 
11721 // This pattern is automatically generated from aarch64_ad.m4
11722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11723 // val ^ (-1 ^ (val >> shift)) ==> eon
11724 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11725                          iRegL src1, iRegL src2,
11726                          immI src3, immL_M1 src4) %{
11727   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11728   ins_cost(1.9 * INSN_COST);
11729   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11730 
11731   ins_encode %{
11732     __ eon(as_Register($dst$$reg),
11733               as_Register($src1$$reg),
11734               as_Register($src2$$reg),
11735               Assembler::ASR,
11736               $src3$$constant & 0x3f);
11737   %}
11738 
11739   ins_pipe(ialu_reg_reg_shift);
11740 %}
11741 
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 // val ^ (-1 ^ (val ror shift)) ==> eonw
11745 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11746                          iRegIorL2I src1, iRegIorL2I src2,
11747                          immI src3, immI_M1 src4) %{
11748   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11749   ins_cost(1.9 * INSN_COST);
11750   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11751 
11752   ins_encode %{
11753     __ eonw(as_Register($dst$$reg),
11754               as_Register($src1$$reg),
11755               as_Register($src2$$reg),
11756               Assembler::ROR,
11757               $src3$$constant & 0x1f);
11758   %}
11759 
11760   ins_pipe(ialu_reg_reg_shift);
11761 %}
11762 
11763 // This pattern is automatically generated from aarch64_ad.m4
11764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11765 // val ^ (-1 ^ (val ror shift)) ==> eon
11766 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11767                          iRegL src1, iRegL src2,
11768                          immI src3, immL_M1 src4) %{
11769   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11770   ins_cost(1.9 * INSN_COST);
11771   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11772 
11773   ins_encode %{
11774     __ eon(as_Register($dst$$reg),
11775               as_Register($src1$$reg),
11776               as_Register($src2$$reg),
11777               Assembler::ROR,
11778               $src3$$constant & 0x3f);
11779   %}
11780 
11781   ins_pipe(ialu_reg_reg_shift);
11782 %}
11783 
11784 // This pattern is automatically generated from aarch64_ad.m4
11785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11786 // val ^ (-1 ^ (val << shift)) ==> eonw
11787 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11788                          iRegIorL2I src1, iRegIorL2I src2,
11789                          immI src3, immI_M1 src4) %{
11790   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11791   ins_cost(1.9 * INSN_COST);
11792   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11793 
11794   ins_encode %{
11795     __ eonw(as_Register($dst$$reg),
11796               as_Register($src1$$reg),
11797               as_Register($src2$$reg),
11798               Assembler::LSL,
11799               $src3$$constant & 0x1f);
11800   %}
11801 
11802   ins_pipe(ialu_reg_reg_shift);
11803 %}
11804 
11805 // This pattern is automatically generated from aarch64_ad.m4
11806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11807 // val ^ (-1 ^ (val << shift)) ==> eon
11808 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11809                          iRegL src1, iRegL src2,
11810                          immI src3, immL_M1 src4) %{
11811   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11812   ins_cost(1.9 * INSN_COST);
11813   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11814 
11815   ins_encode %{
11816     __ eon(as_Register($dst$$reg),
11817               as_Register($src1$$reg),
11818               as_Register($src2$$reg),
11819               Assembler::LSL,
11820               $src3$$constant & 0x3f);
11821   %}
11822 
11823   ins_pipe(ialu_reg_reg_shift);
11824 %}
11825 
11826 // This pattern is automatically generated from aarch64_ad.m4
11827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11828 // val | (-1 ^ (val >>> shift)) ==> ornw
11829 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11830                          iRegIorL2I src1, iRegIorL2I src2,
11831                          immI src3, immI_M1 src4) %{
11832   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11833   ins_cost(1.9 * INSN_COST);
11834   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11835 
11836   ins_encode %{
11837     __ ornw(as_Register($dst$$reg),
11838               as_Register($src1$$reg),
11839               as_Register($src2$$reg),
11840               Assembler::LSR,
11841               $src3$$constant & 0x1f);
11842   %}
11843 
11844   ins_pipe(ialu_reg_reg_shift);
11845 %}
11846 
11847 // This pattern is automatically generated from aarch64_ad.m4
11848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11849 // val | (-1 ^ (val >>> shift)) ==> orn
11850 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11851                          iRegL src1, iRegL src2,
11852                          immI src3, immL_M1 src4) %{
11853   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11854   ins_cost(1.9 * INSN_COST);
11855   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11856 
11857   ins_encode %{
11858     __ orn(as_Register($dst$$reg),
11859               as_Register($src1$$reg),
11860               as_Register($src2$$reg),
11861               Assembler::LSR,
11862               $src3$$constant & 0x3f);
11863   %}
11864 
11865   ins_pipe(ialu_reg_reg_shift);
11866 %}
11867 
11868 // This pattern is automatically generated from aarch64_ad.m4
11869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11870 // val | (-1 ^ (val >> shift)) ==> ornw
11871 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11872                          iRegIorL2I src1, iRegIorL2I src2,
11873                          immI src3, immI_M1 src4) %{
11874   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11875   ins_cost(1.9 * INSN_COST);
11876   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11877 
11878   ins_encode %{
11879     __ ornw(as_Register($dst$$reg),
11880               as_Register($src1$$reg),
11881               as_Register($src2$$reg),
11882               Assembler::ASR,
11883               $src3$$constant & 0x1f);
11884   %}
11885 
11886   ins_pipe(ialu_reg_reg_shift);
11887 %}
11888 
11889 // This pattern is automatically generated from aarch64_ad.m4
11890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11891 // val | (-1 ^ (val >> shift)) ==> orn
11892 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11893                          iRegL src1, iRegL src2,
11894                          immI src3, immL_M1 src4) %{
11895   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11896   ins_cost(1.9 * INSN_COST);
11897   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11898 
11899   ins_encode %{
11900     __ orn(as_Register($dst$$reg),
11901               as_Register($src1$$reg),
11902               as_Register($src2$$reg),
11903               Assembler::ASR,
11904               $src3$$constant & 0x3f);
11905   %}
11906 
11907   ins_pipe(ialu_reg_reg_shift);
11908 %}
11909 
11910 // This pattern is automatically generated from aarch64_ad.m4
11911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11912 // val | (-1 ^ (val ror shift)) ==> ornw
11913 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11914                          iRegIorL2I src1, iRegIorL2I src2,
11915                          immI src3, immI_M1 src4) %{
11916   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11917   ins_cost(1.9 * INSN_COST);
11918   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11919 
11920   ins_encode %{
11921     __ ornw(as_Register($dst$$reg),
11922               as_Register($src1$$reg),
11923               as_Register($src2$$reg),
11924               Assembler::ROR,
11925               $src3$$constant & 0x1f);
11926   %}
11927 
11928   ins_pipe(ialu_reg_reg_shift);
11929 %}
11930 
11931 // This pattern is automatically generated from aarch64_ad.m4
11932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11933 // val | (-1 ^ (val ror shift)) ==> orn
11934 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11935                          iRegL src1, iRegL src2,
11936                          immI src3, immL_M1 src4) %{
11937   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11938   ins_cost(1.9 * INSN_COST);
11939   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11940 
11941   ins_encode %{
11942     __ orn(as_Register($dst$$reg),
11943               as_Register($src1$$reg),
11944               as_Register($src2$$reg),
11945               Assembler::ROR,
11946               $src3$$constant & 0x3f);
11947   %}
11948 
11949   ins_pipe(ialu_reg_reg_shift);
11950 %}
11951 
11952 // This pattern is automatically generated from aarch64_ad.m4
11953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11954 // val | (-1 ^ (val << shift)) ==> ornw
11955 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11956                          iRegIorL2I src1, iRegIorL2I src2,
11957                          immI src3, immI_M1 src4) %{
11958   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11959   ins_cost(1.9 * INSN_COST);
11960   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11961 
11962   ins_encode %{
11963     __ ornw(as_Register($dst$$reg),
11964               as_Register($src1$$reg),
11965               as_Register($src2$$reg),
11966               Assembler::LSL,
11967               $src3$$constant & 0x1f);
11968   %}
11969 
11970   ins_pipe(ialu_reg_reg_shift);
11971 %}
11972 
11973 // This pattern is automatically generated from aarch64_ad.m4
11974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11975 // val | (-1 ^ (val << shift)) ==> orn
11976 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11977                          iRegL src1, iRegL src2,
11978                          immI src3, immL_M1 src4) %{
11979   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11980   ins_cost(1.9 * INSN_COST);
11981   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11982 
11983   ins_encode %{
11984     __ orn(as_Register($dst$$reg),
11985               as_Register($src1$$reg),
11986               as_Register($src2$$reg),
11987               Assembler::LSL,
11988               $src3$$constant & 0x3f);
11989   %}
11990 
11991   ins_pipe(ialu_reg_reg_shift);
11992 %}
11993 
11994 // This pattern is automatically generated from aarch64_ad.m4
11995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11996 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11997                          iRegIorL2I src1, iRegIorL2I src2,
11998                          immI src3) %{
11999   match(Set dst (AndI src1 (URShiftI src2 src3)));
12000 
12001   ins_cost(1.9 * INSN_COST);
12002   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12003 
12004   ins_encode %{
12005     __ andw(as_Register($dst$$reg),
12006               as_Register($src1$$reg),
12007               as_Register($src2$$reg),
12008               Assembler::LSR,
12009               $src3$$constant & 0x1f);
12010   %}
12011 
12012   ins_pipe(ialu_reg_reg_shift);
12013 %}
12014 
12015 // This pattern is automatically generated from aarch64_ad.m4
12016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12017 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12018                          iRegL src1, iRegL src2,
12019                          immI src3) %{
12020   match(Set dst (AndL src1 (URShiftL src2 src3)));
12021 
12022   ins_cost(1.9 * INSN_COST);
12023   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12024 
12025   ins_encode %{
12026     __ andr(as_Register($dst$$reg),
12027               as_Register($src1$$reg),
12028               as_Register($src2$$reg),
12029               Assembler::LSR,
12030               $src3$$constant & 0x3f);
12031   %}
12032 
12033   ins_pipe(ialu_reg_reg_shift);
12034 %}
12035 
12036 // This pattern is automatically generated from aarch64_ad.m4
12037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12038 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12039                          iRegIorL2I src1, iRegIorL2I src2,
12040                          immI src3) %{
12041   match(Set dst (AndI src1 (RShiftI src2 src3)));
12042 
12043   ins_cost(1.9 * INSN_COST);
12044   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12045 
12046   ins_encode %{
12047     __ andw(as_Register($dst$$reg),
12048               as_Register($src1$$reg),
12049               as_Register($src2$$reg),
12050               Assembler::ASR,
12051               $src3$$constant & 0x1f);
12052   %}
12053 
12054   ins_pipe(ialu_reg_reg_shift);
12055 %}
12056 
12057 // This pattern is automatically generated from aarch64_ad.m4
12058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12059 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12060                          iRegL src1, iRegL src2,
12061                          immI src3) %{
12062   match(Set dst (AndL src1 (RShiftL src2 src3)));
12063 
12064   ins_cost(1.9 * INSN_COST);
12065   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12066 
12067   ins_encode %{
12068     __ andr(as_Register($dst$$reg),
12069               as_Register($src1$$reg),
12070               as_Register($src2$$reg),
12071               Assembler::ASR,
12072               $src3$$constant & 0x3f);
12073   %}
12074 
12075   ins_pipe(ialu_reg_reg_shift);
12076 %}
12077 
12078 // This pattern is automatically generated from aarch64_ad.m4
12079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12080 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12081                          iRegIorL2I src1, iRegIorL2I src2,
12082                          immI src3) %{
12083   match(Set dst (AndI src1 (LShiftI src2 src3)));
12084 
12085   ins_cost(1.9 * INSN_COST);
12086   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12087 
12088   ins_encode %{
12089     __ andw(as_Register($dst$$reg),
12090               as_Register($src1$$reg),
12091               as_Register($src2$$reg),
12092               Assembler::LSL,
12093               $src3$$constant & 0x1f);
12094   %}
12095 
12096   ins_pipe(ialu_reg_reg_shift);
12097 %}
12098 
12099 // This pattern is automatically generated from aarch64_ad.m4
12100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12101 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12102                          iRegL src1, iRegL src2,
12103                          immI src3) %{
12104   match(Set dst (AndL src1 (LShiftL src2 src3)));
12105 
12106   ins_cost(1.9 * INSN_COST);
12107   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12108 
12109   ins_encode %{
12110     __ andr(as_Register($dst$$reg),
12111               as_Register($src1$$reg),
12112               as_Register($src2$$reg),
12113               Assembler::LSL,
12114               $src3$$constant & 0x3f);
12115   %}
12116 
12117   ins_pipe(ialu_reg_reg_shift);
12118 %}
12119 
12120 // This pattern is automatically generated from aarch64_ad.m4
12121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12122 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12123                          iRegIorL2I src1, iRegIorL2I src2,
12124                          immI src3) %{
12125   match(Set dst (AndI src1 (RotateRight src2 src3)));
12126 
12127   ins_cost(1.9 * INSN_COST);
12128   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12129 
12130   ins_encode %{
12131     __ andw(as_Register($dst$$reg),
12132               as_Register($src1$$reg),
12133               as_Register($src2$$reg),
12134               Assembler::ROR,
12135               $src3$$constant & 0x1f);
12136   %}
12137 
12138   ins_pipe(ialu_reg_reg_shift);
12139 %}
12140 
12141 // This pattern is automatically generated from aarch64_ad.m4
12142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12143 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12144                          iRegL src1, iRegL src2,
12145                          immI src3) %{
12146   match(Set dst (AndL src1 (RotateRight src2 src3)));
12147 
12148   ins_cost(1.9 * INSN_COST);
12149   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12150 
12151   ins_encode %{
12152     __ andr(as_Register($dst$$reg),
12153               as_Register($src1$$reg),
12154               as_Register($src2$$reg),
12155               Assembler::ROR,
12156               $src3$$constant & 0x3f);
12157   %}
12158 
12159   ins_pipe(ialu_reg_reg_shift);
12160 %}
12161 
12162 // This pattern is automatically generated from aarch64_ad.m4
12163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12164 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12165                          iRegIorL2I src1, iRegIorL2I src2,
12166                          immI src3) %{
12167   match(Set dst (XorI src1 (URShiftI src2 src3)));
12168 
12169   ins_cost(1.9 * INSN_COST);
12170   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12171 
12172   ins_encode %{
12173     __ eorw(as_Register($dst$$reg),
12174               as_Register($src1$$reg),
12175               as_Register($src2$$reg),
12176               Assembler::LSR,
12177               $src3$$constant & 0x1f);
12178   %}
12179 
12180   ins_pipe(ialu_reg_reg_shift);
12181 %}
12182 
12183 // This pattern is automatically generated from aarch64_ad.m4
12184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12185 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12186                          iRegL src1, iRegL src2,
12187                          immI src3) %{
12188   match(Set dst (XorL src1 (URShiftL src2 src3)));
12189 
12190   ins_cost(1.9 * INSN_COST);
12191   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12192 
12193   ins_encode %{
12194     __ eor(as_Register($dst$$reg),
12195               as_Register($src1$$reg),
12196               as_Register($src2$$reg),
12197               Assembler::LSR,
12198               $src3$$constant & 0x3f);
12199   %}
12200 
12201   ins_pipe(ialu_reg_reg_shift);
12202 %}
12203 
12204 // This pattern is automatically generated from aarch64_ad.m4
12205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12206 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12207                          iRegIorL2I src1, iRegIorL2I src2,
12208                          immI src3) %{
12209   match(Set dst (XorI src1 (RShiftI src2 src3)));
12210 
12211   ins_cost(1.9 * INSN_COST);
12212   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12213 
12214   ins_encode %{
12215     __ eorw(as_Register($dst$$reg),
12216               as_Register($src1$$reg),
12217               as_Register($src2$$reg),
12218               Assembler::ASR,
12219               $src3$$constant & 0x1f);
12220   %}
12221 
12222   ins_pipe(ialu_reg_reg_shift);
12223 %}
12224 
12225 // This pattern is automatically generated from aarch64_ad.m4
12226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12227 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12228                          iRegL src1, iRegL src2,
12229                          immI src3) %{
12230   match(Set dst (XorL src1 (RShiftL src2 src3)));
12231 
12232   ins_cost(1.9 * INSN_COST);
12233   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12234 
12235   ins_encode %{
12236     __ eor(as_Register($dst$$reg),
12237               as_Register($src1$$reg),
12238               as_Register($src2$$reg),
12239               Assembler::ASR,
12240               $src3$$constant & 0x3f);
12241   %}
12242 
12243   ins_pipe(ialu_reg_reg_shift);
12244 %}
12245 
12246 // This pattern is automatically generated from aarch64_ad.m4
12247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12248 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12249                          iRegIorL2I src1, iRegIorL2I src2,
12250                          immI src3) %{
12251   match(Set dst (XorI src1 (LShiftI src2 src3)));
12252 
12253   ins_cost(1.9 * INSN_COST);
12254   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12255 
12256   ins_encode %{
12257     __ eorw(as_Register($dst$$reg),
12258               as_Register($src1$$reg),
12259               as_Register($src2$$reg),
12260               Assembler::LSL,
12261               $src3$$constant & 0x1f);
12262   %}
12263 
12264   ins_pipe(ialu_reg_reg_shift);
12265 %}
12266 
12267 // This pattern is automatically generated from aarch64_ad.m4
12268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12269 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12270                          iRegL src1, iRegL src2,
12271                          immI src3) %{
12272   match(Set dst (XorL src1 (LShiftL src2 src3)));
12273 
12274   ins_cost(1.9 * INSN_COST);
12275   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12276 
12277   ins_encode %{
12278     __ eor(as_Register($dst$$reg),
12279               as_Register($src1$$reg),
12280               as_Register($src2$$reg),
12281               Assembler::LSL,
12282               $src3$$constant & 0x3f);
12283   %}
12284 
12285   ins_pipe(ialu_reg_reg_shift);
12286 %}
12287 
12288 // This pattern is automatically generated from aarch64_ad.m4
12289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12290 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12291                          iRegIorL2I src1, iRegIorL2I src2,
12292                          immI src3) %{
12293   match(Set dst (XorI src1 (RotateRight src2 src3)));
12294 
12295   ins_cost(1.9 * INSN_COST);
12296   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12297 
12298   ins_encode %{
12299     __ eorw(as_Register($dst$$reg),
12300               as_Register($src1$$reg),
12301               as_Register($src2$$reg),
12302               Assembler::ROR,
12303               $src3$$constant & 0x1f);
12304   %}
12305 
12306   ins_pipe(ialu_reg_reg_shift);
12307 %}
12308 
12309 // This pattern is automatically generated from aarch64_ad.m4
12310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12311 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12312                          iRegL src1, iRegL src2,
12313                          immI src3) %{
12314   match(Set dst (XorL src1 (RotateRight src2 src3)));
12315 
12316   ins_cost(1.9 * INSN_COST);
12317   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12318 
12319   ins_encode %{
12320     __ eor(as_Register($dst$$reg),
12321               as_Register($src1$$reg),
12322               as_Register($src2$$reg),
12323               Assembler::ROR,
12324               $src3$$constant & 0x3f);
12325   %}
12326 
12327   ins_pipe(ialu_reg_reg_shift);
12328 %}
12329 
12330 // This pattern is automatically generated from aarch64_ad.m4
12331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12332 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12333                          iRegIorL2I src1, iRegIorL2I src2,
12334                          immI src3) %{
12335   match(Set dst (OrI src1 (URShiftI src2 src3)));
12336 
12337   ins_cost(1.9 * INSN_COST);
12338   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12339 
12340   ins_encode %{
12341     __ orrw(as_Register($dst$$reg),
12342               as_Register($src1$$reg),
12343               as_Register($src2$$reg),
12344               Assembler::LSR,
12345               $src3$$constant & 0x1f);
12346   %}
12347 
12348   ins_pipe(ialu_reg_reg_shift);
12349 %}
12350 
12351 // This pattern is automatically generated from aarch64_ad.m4
12352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12353 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12354                          iRegL src1, iRegL src2,
12355                          immI src3) %{
12356   match(Set dst (OrL src1 (URShiftL src2 src3)));
12357 
12358   ins_cost(1.9 * INSN_COST);
12359   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12360 
12361   ins_encode %{
12362     __ orr(as_Register($dst$$reg),
12363               as_Register($src1$$reg),
12364               as_Register($src2$$reg),
12365               Assembler::LSR,
12366               $src3$$constant & 0x3f);
12367   %}
12368 
12369   ins_pipe(ialu_reg_reg_shift);
12370 %}
12371 
12372 // This pattern is automatically generated from aarch64_ad.m4
12373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12374 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12375                          iRegIorL2I src1, iRegIorL2I src2,
12376                          immI src3) %{
12377   match(Set dst (OrI src1 (RShiftI src2 src3)));
12378 
12379   ins_cost(1.9 * INSN_COST);
12380   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12381 
12382   ins_encode %{
12383     __ orrw(as_Register($dst$$reg),
12384               as_Register($src1$$reg),
12385               as_Register($src2$$reg),
12386               Assembler::ASR,
12387               $src3$$constant & 0x1f);
12388   %}
12389 
12390   ins_pipe(ialu_reg_reg_shift);
12391 %}
12392 
12393 // This pattern is automatically generated from aarch64_ad.m4
12394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12395 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12396                          iRegL src1, iRegL src2,
12397                          immI src3) %{
12398   match(Set dst (OrL src1 (RShiftL src2 src3)));
12399 
12400   ins_cost(1.9 * INSN_COST);
12401   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12402 
12403   ins_encode %{
12404     __ orr(as_Register($dst$$reg),
12405               as_Register($src1$$reg),
12406               as_Register($src2$$reg),
12407               Assembler::ASR,
12408               $src3$$constant & 0x3f);
12409   %}
12410 
12411   ins_pipe(ialu_reg_reg_shift);
12412 %}
12413 
12414 // This pattern is automatically generated from aarch64_ad.m4
12415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12416 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12417                          iRegIorL2I src1, iRegIorL2I src2,
12418                          immI src3) %{
12419   match(Set dst (OrI src1 (LShiftI src2 src3)));
12420 
12421   ins_cost(1.9 * INSN_COST);
12422   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12423 
12424   ins_encode %{
12425     __ orrw(as_Register($dst$$reg),
12426               as_Register($src1$$reg),
12427               as_Register($src2$$reg),
12428               Assembler::LSL,
12429               $src3$$constant & 0x1f);
12430   %}
12431 
12432   ins_pipe(ialu_reg_reg_shift);
12433 %}
12434 
12435 // This pattern is automatically generated from aarch64_ad.m4
12436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12437 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12438                          iRegL src1, iRegL src2,
12439                          immI src3) %{
12440   match(Set dst (OrL src1 (LShiftL src2 src3)));
12441 
12442   ins_cost(1.9 * INSN_COST);
12443   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12444 
12445   ins_encode %{
12446     __ orr(as_Register($dst$$reg),
12447               as_Register($src1$$reg),
12448               as_Register($src2$$reg),
12449               Assembler::LSL,
12450               $src3$$constant & 0x3f);
12451   %}
12452 
12453   ins_pipe(ialu_reg_reg_shift);
12454 %}
12455 
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12459                          iRegIorL2I src1, iRegIorL2I src2,
12460                          immI src3) %{
12461   match(Set dst (OrI src1 (RotateRight src2 src3)));
12462 
12463   ins_cost(1.9 * INSN_COST);
12464   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12465 
12466   ins_encode %{
12467     __ orrw(as_Register($dst$$reg),
12468               as_Register($src1$$reg),
12469               as_Register($src2$$reg),
12470               Assembler::ROR,
12471               $src3$$constant & 0x1f);
12472   %}
12473 
12474   ins_pipe(ialu_reg_reg_shift);
12475 %}
12476 
12477 // This pattern is automatically generated from aarch64_ad.m4
12478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12479 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12480                          iRegL src1, iRegL src2,
12481                          immI src3) %{
12482   match(Set dst (OrL src1 (RotateRight src2 src3)));
12483 
12484   ins_cost(1.9 * INSN_COST);
12485   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12486 
12487   ins_encode %{
12488     __ orr(as_Register($dst$$reg),
12489               as_Register($src1$$reg),
12490               as_Register($src2$$reg),
12491               Assembler::ROR,
12492               $src3$$constant & 0x3f);
12493   %}
12494 
12495   ins_pipe(ialu_reg_reg_shift);
12496 %}
12497 
12498 // This pattern is automatically generated from aarch64_ad.m4
12499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12500 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12501                          iRegIorL2I src1, iRegIorL2I src2,
12502                          immI src3) %{
12503   match(Set dst (AddI src1 (URShiftI src2 src3)));
12504 
12505   ins_cost(1.9 * INSN_COST);
12506   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12507 
12508   ins_encode %{
12509     __ addw(as_Register($dst$$reg),
12510               as_Register($src1$$reg),
12511               as_Register($src2$$reg),
12512               Assembler::LSR,
12513               $src3$$constant & 0x1f);
12514   %}
12515 
12516   ins_pipe(ialu_reg_reg_shift);
12517 %}
12518 
12519 // This pattern is automatically generated from aarch64_ad.m4
12520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12521 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12522                          iRegL src1, iRegL src2,
12523                          immI src3) %{
12524   match(Set dst (AddL src1 (URShiftL src2 src3)));
12525 
12526   ins_cost(1.9 * INSN_COST);
12527   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12528 
12529   ins_encode %{
12530     __ add(as_Register($dst$$reg),
12531               as_Register($src1$$reg),
12532               as_Register($src2$$reg),
12533               Assembler::LSR,
12534               $src3$$constant & 0x3f);
12535   %}
12536 
12537   ins_pipe(ialu_reg_reg_shift);
12538 %}
12539 
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12543                          iRegIorL2I src1, iRegIorL2I src2,
12544                          immI src3) %{
12545   match(Set dst (AddI src1 (RShiftI src2 src3)));
12546 
12547   ins_cost(1.9 * INSN_COST);
12548   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12549 
12550   ins_encode %{
12551     __ addw(as_Register($dst$$reg),
12552               as_Register($src1$$reg),
12553               as_Register($src2$$reg),
12554               Assembler::ASR,
12555               $src3$$constant & 0x1f);
12556   %}
12557 
12558   ins_pipe(ialu_reg_reg_shift);
12559 %}
12560 
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12564                          iRegL src1, iRegL src2,
12565                          immI src3) %{
12566   match(Set dst (AddL src1 (RShiftL src2 src3)));
12567 
12568   ins_cost(1.9 * INSN_COST);
12569   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12570 
12571   ins_encode %{
12572     __ add(as_Register($dst$$reg),
12573               as_Register($src1$$reg),
12574               as_Register($src2$$reg),
12575               Assembler::ASR,
12576               $src3$$constant & 0x3f);
12577   %}
12578 
12579   ins_pipe(ialu_reg_reg_shift);
12580 %}
12581 
12582 // This pattern is automatically generated from aarch64_ad.m4
12583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12584 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12585                          iRegIorL2I src1, iRegIorL2I src2,
12586                          immI src3) %{
12587   match(Set dst (AddI src1 (LShiftI src2 src3)));
12588 
12589   ins_cost(1.9 * INSN_COST);
12590   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12591 
12592   ins_encode %{
12593     __ addw(as_Register($dst$$reg),
12594               as_Register($src1$$reg),
12595               as_Register($src2$$reg),
12596               Assembler::LSL,
12597               $src3$$constant & 0x1f);
12598   %}
12599 
12600   ins_pipe(ialu_reg_reg_shift);
12601 %}
12602 
12603 // This pattern is automatically generated from aarch64_ad.m4
12604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12605 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12606                          iRegL src1, iRegL src2,
12607                          immI src3) %{
12608   match(Set dst (AddL src1 (LShiftL src2 src3)));
12609 
12610   ins_cost(1.9 * INSN_COST);
12611   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12612 
12613   ins_encode %{
12614     __ add(as_Register($dst$$reg),
12615               as_Register($src1$$reg),
12616               as_Register($src2$$reg),
12617               Assembler::LSL,
12618               $src3$$constant & 0x3f);
12619   %}
12620 
12621   ins_pipe(ialu_reg_reg_shift);
12622 %}
12623 
12624 // This pattern is automatically generated from aarch64_ad.m4
12625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12626 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12627                          iRegIorL2I src1, iRegIorL2I src2,
12628                          immI src3) %{
12629   match(Set dst (SubI src1 (URShiftI src2 src3)));
12630 
12631   ins_cost(1.9 * INSN_COST);
12632   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12633 
12634   ins_encode %{
12635     __ subw(as_Register($dst$$reg),
12636               as_Register($src1$$reg),
12637               as_Register($src2$$reg),
12638               Assembler::LSR,
12639               $src3$$constant & 0x1f);
12640   %}
12641 
12642   ins_pipe(ialu_reg_reg_shift);
12643 %}
12644 
12645 // This pattern is automatically generated from aarch64_ad.m4
12646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12647 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12648                          iRegL src1, iRegL src2,
12649                          immI src3) %{
12650   match(Set dst (SubL src1 (URShiftL src2 src3)));
12651 
12652   ins_cost(1.9 * INSN_COST);
12653   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12654 
12655   ins_encode %{
12656     __ sub(as_Register($dst$$reg),
12657               as_Register($src1$$reg),
12658               as_Register($src2$$reg),
12659               Assembler::LSR,
12660               $src3$$constant & 0x3f);
12661   %}
12662 
12663   ins_pipe(ialu_reg_reg_shift);
12664 %}
12665 
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12669                          iRegIorL2I src1, iRegIorL2I src2,
12670                          immI src3) %{
12671   match(Set dst (SubI src1 (RShiftI src2 src3)));
12672 
12673   ins_cost(1.9 * INSN_COST);
12674   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12675 
12676   ins_encode %{
12677     __ subw(as_Register($dst$$reg),
12678               as_Register($src1$$reg),
12679               as_Register($src2$$reg),
12680               Assembler::ASR,
12681               $src3$$constant & 0x1f);
12682   %}
12683 
12684   ins_pipe(ialu_reg_reg_shift);
12685 %}
12686 
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12690                          iRegL src1, iRegL src2,
12691                          immI src3) %{
12692   match(Set dst (SubL src1 (RShiftL src2 src3)));
12693 
12694   ins_cost(1.9 * INSN_COST);
12695   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12696 
12697   ins_encode %{
12698     __ sub(as_Register($dst$$reg),
12699               as_Register($src1$$reg),
12700               as_Register($src2$$reg),
12701               Assembler::ASR,
12702               $src3$$constant & 0x3f);
12703   %}
12704 
12705   ins_pipe(ialu_reg_reg_shift);
12706 %}
12707 
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12711                          iRegIorL2I src1, iRegIorL2I src2,
12712                          immI src3) %{
12713   match(Set dst (SubI src1 (LShiftI src2 src3)));
12714 
12715   ins_cost(1.9 * INSN_COST);
12716   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12717 
12718   ins_encode %{
12719     __ subw(as_Register($dst$$reg),
12720               as_Register($src1$$reg),
12721               as_Register($src2$$reg),
12722               Assembler::LSL,
12723               $src3$$constant & 0x1f);
12724   %}
12725 
12726   ins_pipe(ialu_reg_reg_shift);
12727 %}
12728 
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12732                          iRegL src1, iRegL src2,
12733                          immI src3) %{
12734   match(Set dst (SubL src1 (LShiftL src2 src3)));
12735 
12736   ins_cost(1.9 * INSN_COST);
12737   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12738 
12739   ins_encode %{
12740     __ sub(as_Register($dst$$reg),
12741               as_Register($src1$$reg),
12742               as_Register($src2$$reg),
12743               Assembler::LSL,
12744               $src3$$constant & 0x3f);
12745   %}
12746 
12747   ins_pipe(ialu_reg_reg_shift);
12748 %}
12749 
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 
12753 // Shift Left followed by Shift Right.
12754 // This idiom is used by the compiler for the i2b bytecode etc.
12755 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12756 %{
12757   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12758   ins_cost(INSN_COST * 2);
12759   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12760   ins_encode %{
12761     int lshift = $lshift_count$$constant & 63;
12762     int rshift = $rshift_count$$constant & 63;
12763     int s = 63 - lshift;
12764     int r = (rshift - lshift) & 63;
12765     __ sbfm(as_Register($dst$$reg),
12766             as_Register($src$$reg),
12767             r, s);
12768   %}
12769 
12770   ins_pipe(ialu_reg_shift);
12771 %}
12772 
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 
12776 // Shift Left followed by Shift Right.
12777 // This idiom is used by the compiler for the i2b bytecode etc.
12778 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12779 %{
12780   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12781   ins_cost(INSN_COST * 2);
12782   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12783   ins_encode %{
12784     int lshift = $lshift_count$$constant & 31;
12785     int rshift = $rshift_count$$constant & 31;
12786     int s = 31 - lshift;
12787     int r = (rshift - lshift) & 31;
12788     __ sbfmw(as_Register($dst$$reg),
12789             as_Register($src$$reg),
12790             r, s);
12791   %}
12792 
12793   ins_pipe(ialu_reg_shift);
12794 %}
12795 
12796 // This pattern is automatically generated from aarch64_ad.m4
12797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12798 
12799 // Shift Left followed by Shift Right.
12800 // This idiom is used by the compiler for the i2b bytecode etc.
12801 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12802 %{
12803   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12804   ins_cost(INSN_COST * 2);
12805   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12806   ins_encode %{
12807     int lshift = $lshift_count$$constant & 63;
12808     int rshift = $rshift_count$$constant & 63;
12809     int s = 63 - lshift;
12810     int r = (rshift - lshift) & 63;
12811     __ ubfm(as_Register($dst$$reg),
12812             as_Register($src$$reg),
12813             r, s);
12814   %}
12815 
12816   ins_pipe(ialu_reg_shift);
12817 %}
12818 
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821 
12822 // Shift Left followed by Shift Right.
12823 // This idiom is used by the compiler for the i2b bytecode etc.
12824 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12825 %{
12826   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12827   ins_cost(INSN_COST * 2);
12828   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12829   ins_encode %{
12830     int lshift = $lshift_count$$constant & 31;
12831     int rshift = $rshift_count$$constant & 31;
12832     int s = 31 - lshift;
12833     int r = (rshift - lshift) & 31;
12834     __ ubfmw(as_Register($dst$$reg),
12835             as_Register($src$$reg),
12836             r, s);
12837   %}
12838 
12839   ins_pipe(ialu_reg_shift);
12840 %}
12841 
12842 // Bitfield extract with shift & mask
12843 
12844 // This pattern is automatically generated from aarch64_ad.m4
12845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12846 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12847 %{
12848   match(Set dst (AndI (URShiftI src rshift) mask));
12849   // Make sure we are not going to exceed what ubfxw can do.
12850   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12851 
12852   ins_cost(INSN_COST);
12853   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12854   ins_encode %{
12855     int rshift = $rshift$$constant & 31;
12856     intptr_t mask = $mask$$constant;
12857     int width = exact_log2(mask+1);
12858     __ ubfxw(as_Register($dst$$reg),
12859             as_Register($src$$reg), rshift, width);
12860   %}
12861   ins_pipe(ialu_reg_shift);
12862 %}
12863 
12864 // This pattern is automatically generated from aarch64_ad.m4
12865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12866 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12867 %{
12868   match(Set dst (AndL (URShiftL src rshift) mask));
12869   // Make sure we are not going to exceed what ubfx can do.
12870   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12871 
12872   ins_cost(INSN_COST);
12873   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12874   ins_encode %{
12875     int rshift = $rshift$$constant & 63;
12876     intptr_t mask = $mask$$constant;
12877     int width = exact_log2_long(mask+1);
12878     __ ubfx(as_Register($dst$$reg),
12879             as_Register($src$$reg), rshift, width);
12880   %}
12881   ins_pipe(ialu_reg_shift);
12882 %}
12883 
12884 
12885 // This pattern is automatically generated from aarch64_ad.m4
12886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12887 
12888 // We can use ubfx when extending an And with a mask when we know mask
12889 // is positive.  We know that because immI_bitmask guarantees it.
12890 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12891 %{
12892   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12893   // Make sure we are not going to exceed what ubfxw can do.
12894   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12895 
12896   ins_cost(INSN_COST * 2);
12897   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12898   ins_encode %{
12899     int rshift = $rshift$$constant & 31;
12900     intptr_t mask = $mask$$constant;
12901     int width = exact_log2(mask+1);
12902     __ ubfx(as_Register($dst$$reg),
12903             as_Register($src$$reg), rshift, width);
12904   %}
12905   ins_pipe(ialu_reg_shift);
12906 %}
12907 
12908 
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 
12912 // We can use ubfiz when masking by a positive number and then left shifting the result.
12913 // We know that the mask is positive because immI_bitmask guarantees it.
12914 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12915 %{
12916   match(Set dst (LShiftI (AndI src mask) lshift));
12917   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12918 
12919   ins_cost(INSN_COST);
12920   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12921   ins_encode %{
12922     int lshift = $lshift$$constant & 31;
12923     intptr_t mask = $mask$$constant;
12924     int width = exact_log2(mask+1);
12925     __ ubfizw(as_Register($dst$$reg),
12926           as_Register($src$$reg), lshift, width);
12927   %}
12928   ins_pipe(ialu_reg_shift);
12929 %}
12930 
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 
12934 // We can use ubfiz when masking by a positive number and then left shifting the result.
12935 // We know that the mask is positive because immL_bitmask guarantees it.
12936 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12937 %{
12938   match(Set dst (LShiftL (AndL src mask) lshift));
12939   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12940 
12941   ins_cost(INSN_COST);
12942   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12943   ins_encode %{
12944     int lshift = $lshift$$constant & 63;
12945     intptr_t mask = $mask$$constant;
12946     int width = exact_log2_long(mask+1);
12947     __ ubfiz(as_Register($dst$$reg),
12948           as_Register($src$$reg), lshift, width);
12949   %}
12950   ins_pipe(ialu_reg_shift);
12951 %}
12952 
12953 // This pattern is automatically generated from aarch64_ad.m4
12954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12955 
12956 // We can use ubfiz when masking by a positive number and then left shifting the result.
12957 // We know that the mask is positive because immI_bitmask guarantees it.
12958 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12959 %{
12960   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12961   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12962 
12963   ins_cost(INSN_COST);
12964   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12965   ins_encode %{
12966     int lshift = $lshift$$constant & 31;
12967     intptr_t mask = $mask$$constant;
12968     int width = exact_log2(mask+1);
12969     __ ubfizw(as_Register($dst$$reg),
12970           as_Register($src$$reg), lshift, width);
12971   %}
12972   ins_pipe(ialu_reg_shift);
12973 %}
12974 
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 
12978 // We can use ubfiz when masking by a positive number and then left shifting the result.
12979 // We know that the mask is positive because immL_bitmask guarantees it.
12980 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12981 %{
12982   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12983   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12984 
12985   ins_cost(INSN_COST);
12986   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12987   ins_encode %{
12988     int lshift = $lshift$$constant & 63;
12989     intptr_t mask = $mask$$constant;
12990     int width = exact_log2_long(mask+1);
12991     __ ubfiz(as_Register($dst$$reg),
12992           as_Register($src$$reg), lshift, width);
12993   %}
12994   ins_pipe(ialu_reg_shift);
12995 %}
12996 
12997 
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000 
13001 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13002 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13003 %{
13004   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13005   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13006 
13007   ins_cost(INSN_COST);
13008   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13009   ins_encode %{
13010     int lshift = $lshift$$constant & 63;
13011     intptr_t mask = $mask$$constant;
13012     int width = exact_log2(mask+1);
13013     __ ubfiz(as_Register($dst$$reg),
13014              as_Register($src$$reg), lshift, width);
13015   %}
13016   ins_pipe(ialu_reg_shift);
13017 %}
13018 
13019 // This pattern is automatically generated from aarch64_ad.m4
13020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13021 
13022 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13023 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13024 %{
13025   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13026   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13027 
13028   ins_cost(INSN_COST);
13029   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13030   ins_encode %{
13031     int lshift = $lshift$$constant & 31;
13032     intptr_t mask = $mask$$constant;
13033     int width = exact_log2(mask+1);
13034     __ ubfiz(as_Register($dst$$reg),
13035              as_Register($src$$reg), lshift, width);
13036   %}
13037   ins_pipe(ialu_reg_shift);
13038 %}
13039 
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 
13043 // Can skip int2long conversions after AND with small bitmask
13044 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13045 %{
13046   match(Set dst (ConvI2L (AndI src msk)));
13047   ins_cost(INSN_COST);
13048   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13049   ins_encode %{
13050     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13051   %}
13052   ins_pipe(ialu_reg_shift);
13053 %}
13054 
13055 
13056 // Rotations
13057 // This pattern is automatically generated from aarch64_ad.m4
13058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13059 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13060 %{
13061   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13062   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13063 
13064   ins_cost(INSN_COST);
13065   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13066 
13067   ins_encode %{
13068     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13069             $rshift$$constant & 63);
13070   %}
13071   ins_pipe(ialu_reg_reg_extr);
13072 %}
13073 
13074 
13075 // This pattern is automatically generated from aarch64_ad.m4
13076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13077 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13078 %{
13079   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13080   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13081 
13082   ins_cost(INSN_COST);
13083   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13084 
13085   ins_encode %{
13086     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13087             $rshift$$constant & 31);
13088   %}
13089   ins_pipe(ialu_reg_reg_extr);
13090 %}
13091 
13092 
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13096 %{
13097   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13098   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13099 
13100   ins_cost(INSN_COST);
13101   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13102 
13103   ins_encode %{
13104     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13105             $rshift$$constant & 63);
13106   %}
13107   ins_pipe(ialu_reg_reg_extr);
13108 %}
13109 
13110 
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13114 %{
13115   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13116   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13117 
13118   ins_cost(INSN_COST);
13119   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13120 
13121   ins_encode %{
13122     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13123             $rshift$$constant & 31);
13124   %}
13125   ins_pipe(ialu_reg_reg_extr);
13126 %}
13127 
13128 
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13132 %{
13133   match(Set dst (RotateRight src shift));
13134 
13135   ins_cost(INSN_COST);
13136   format %{ "ror    $dst, $src, $shift" %}
13137 
13138   ins_encode %{
13139      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13140                $shift$$constant & 0x1f);
13141   %}
13142   ins_pipe(ialu_reg_reg_vshift);
13143 %}
13144 
13145 // This pattern is automatically generated from aarch64_ad.m4
13146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13147 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13148 %{
13149   match(Set dst (RotateRight src shift));
13150 
13151   ins_cost(INSN_COST);
13152   format %{ "ror    $dst, $src, $shift" %}
13153 
13154   ins_encode %{
13155      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13156                $shift$$constant & 0x3f);
13157   %}
13158   ins_pipe(ialu_reg_reg_vshift);
13159 %}
13160 
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13164 %{
13165   match(Set dst (RotateRight src shift));
13166 
13167   ins_cost(INSN_COST);
13168   format %{ "ror    $dst, $src, $shift" %}
13169 
13170   ins_encode %{
13171      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13172   %}
13173   ins_pipe(ialu_reg_reg_vshift);
13174 %}
13175 
13176 // This pattern is automatically generated from aarch64_ad.m4
13177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13178 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13179 %{
13180   match(Set dst (RotateRight src shift));
13181 
13182   ins_cost(INSN_COST);
13183   format %{ "ror    $dst, $src, $shift" %}
13184 
13185   ins_encode %{
13186      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13187   %}
13188   ins_pipe(ialu_reg_reg_vshift);
13189 %}
13190 
13191 // This pattern is automatically generated from aarch64_ad.m4
13192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13193 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13194 %{
13195   match(Set dst (RotateLeft src shift));
13196 
13197   ins_cost(INSN_COST);
13198   format %{ "rol    $dst, $src, $shift" %}
13199 
13200   ins_encode %{
13201      __ subw(rscratch1, zr, as_Register($shift$$reg));
13202      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13203   %}
13204   ins_pipe(ialu_reg_reg_vshift);
13205 %}
13206 
13207 // This pattern is automatically generated from aarch64_ad.m4
13208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13209 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13210 %{
13211   match(Set dst (RotateLeft src shift));
13212 
13213   ins_cost(INSN_COST);
13214   format %{ "rol    $dst, $src, $shift" %}
13215 
13216   ins_encode %{
13217      __ subw(rscratch1, zr, as_Register($shift$$reg));
13218      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13219   %}
13220   ins_pipe(ialu_reg_reg_vshift);
13221 %}
13222 
13223 
13224 // Add/subtract (extended)
13225 
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13229 %{
13230   match(Set dst (AddL src1 (ConvI2L src2)));
13231   ins_cost(INSN_COST);
13232   format %{ "add  $dst, $src1, $src2, sxtw" %}
13233 
13234    ins_encode %{
13235      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13236             as_Register($src2$$reg), ext::sxtw);
13237    %}
13238   ins_pipe(ialu_reg_reg);
13239 %}
13240 
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13244 %{
13245   match(Set dst (SubL src1 (ConvI2L src2)));
13246   ins_cost(INSN_COST);
13247   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13248 
13249    ins_encode %{
13250      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13251             as_Register($src2$$reg), ext::sxtw);
13252    %}
13253   ins_pipe(ialu_reg_reg);
13254 %}
13255 
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13259 %{
13260   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13261   ins_cost(INSN_COST);
13262   format %{ "add  $dst, $src1, $src2, sxth" %}
13263 
13264    ins_encode %{
13265      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13266             as_Register($src2$$reg), ext::sxth);
13267    %}
13268   ins_pipe(ialu_reg_reg);
13269 %}
13270 
13271 // This pattern is automatically generated from aarch64_ad.m4
13272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13273 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13274 %{
13275   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13276   ins_cost(INSN_COST);
13277   format %{ "add  $dst, $src1, $src2, sxtb" %}
13278 
13279    ins_encode %{
13280      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13281             as_Register($src2$$reg), ext::sxtb);
13282    %}
13283   ins_pipe(ialu_reg_reg);
13284 %}
13285 
13286 // This pattern is automatically generated from aarch64_ad.m4
13287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13288 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13289 %{
13290   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13291   ins_cost(INSN_COST);
13292   format %{ "add  $dst, $src1, $src2, uxtb" %}
13293 
13294    ins_encode %{
13295      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13296             as_Register($src2$$reg), ext::uxtb);
13297    %}
13298   ins_pipe(ialu_reg_reg);
13299 %}
13300 
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13304 %{
13305   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13306   ins_cost(INSN_COST);
13307   format %{ "add  $dst, $src1, $src2, sxth" %}
13308 
13309    ins_encode %{
13310      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13311             as_Register($src2$$reg), ext::sxth);
13312    %}
13313   ins_pipe(ialu_reg_reg);
13314 %}
13315 
13316 // This pattern is automatically generated from aarch64_ad.m4
13317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13318 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13319 %{
13320   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13321   ins_cost(INSN_COST);
13322   format %{ "add  $dst, $src1, $src2, sxtw" %}
13323 
13324    ins_encode %{
13325      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13326             as_Register($src2$$reg), ext::sxtw);
13327    %}
13328   ins_pipe(ialu_reg_reg);
13329 %}
13330 
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13334 %{
13335   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13336   ins_cost(INSN_COST);
13337   format %{ "add  $dst, $src1, $src2, sxtb" %}
13338 
13339    ins_encode %{
13340      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13341             as_Register($src2$$reg), ext::sxtb);
13342    %}
13343   ins_pipe(ialu_reg_reg);
13344 %}
13345 
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13349 %{
13350   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13351   ins_cost(INSN_COST);
13352   format %{ "add  $dst, $src1, $src2, uxtb" %}
13353 
13354    ins_encode %{
13355      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13356             as_Register($src2$$reg), ext::uxtb);
13357    %}
13358   ins_pipe(ialu_reg_reg);
13359 %}
13360 
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13364 %{
13365   match(Set dst (AddI src1 (AndI src2 mask)));
13366   ins_cost(INSN_COST);
13367   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13368 
13369    ins_encode %{
13370      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13371             as_Register($src2$$reg), ext::uxtb);
13372    %}
13373   ins_pipe(ialu_reg_reg);
13374 %}
13375 
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13379 %{
13380   match(Set dst (AddI src1 (AndI src2 mask)));
13381   ins_cost(INSN_COST);
13382   format %{ "addw  $dst, $src1, $src2, uxth" %}
13383 
13384    ins_encode %{
13385      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13386             as_Register($src2$$reg), ext::uxth);
13387    %}
13388   ins_pipe(ialu_reg_reg);
13389 %}
13390 
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13394 %{
13395   match(Set dst (AddL src1 (AndL src2 mask)));
13396   ins_cost(INSN_COST);
13397   format %{ "add  $dst, $src1, $src2, uxtb" %}
13398 
13399    ins_encode %{
13400      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13401             as_Register($src2$$reg), ext::uxtb);
13402    %}
13403   ins_pipe(ialu_reg_reg);
13404 %}
13405 
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13409 %{
13410   match(Set dst (AddL src1 (AndL src2 mask)));
13411   ins_cost(INSN_COST);
13412   format %{ "add  $dst, $src1, $src2, uxth" %}
13413 
13414    ins_encode %{
13415      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13416             as_Register($src2$$reg), ext::uxth);
13417    %}
13418   ins_pipe(ialu_reg_reg);
13419 %}
13420 
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13424 %{
13425   match(Set dst (AddL src1 (AndL src2 mask)));
13426   ins_cost(INSN_COST);
13427   format %{ "add  $dst, $src1, $src2, uxtw" %}
13428 
13429    ins_encode %{
13430      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13431             as_Register($src2$$reg), ext::uxtw);
13432    %}
13433   ins_pipe(ialu_reg_reg);
13434 %}
13435 
13436 // This pattern is automatically generated from aarch64_ad.m4
13437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13438 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13439 %{
13440   match(Set dst (SubI src1 (AndI src2 mask)));
13441   ins_cost(INSN_COST);
13442   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13443 
13444    ins_encode %{
13445      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13446             as_Register($src2$$reg), ext::uxtb);
13447    %}
13448   ins_pipe(ialu_reg_reg);
13449 %}
13450 
13451 // This pattern is automatically generated from aarch64_ad.m4
13452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13453 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13454 %{
13455   match(Set dst (SubI src1 (AndI src2 mask)));
13456   ins_cost(INSN_COST);
13457   format %{ "subw  $dst, $src1, $src2, uxth" %}
13458 
13459    ins_encode %{
13460      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13461             as_Register($src2$$reg), ext::uxth);
13462    %}
13463   ins_pipe(ialu_reg_reg);
13464 %}
13465 
13466 // This pattern is automatically generated from aarch64_ad.m4
13467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13468 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13469 %{
13470   match(Set dst (SubL src1 (AndL src2 mask)));
13471   ins_cost(INSN_COST);
13472   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13473 
13474    ins_encode %{
13475      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13476             as_Register($src2$$reg), ext::uxtb);
13477    %}
13478   ins_pipe(ialu_reg_reg);
13479 %}
13480 
13481 // This pattern is automatically generated from aarch64_ad.m4
13482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13483 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13484 %{
13485   match(Set dst (SubL src1 (AndL src2 mask)));
13486   ins_cost(INSN_COST);
13487   format %{ "sub  $dst, $src1, $src2, uxth" %}
13488 
13489    ins_encode %{
13490      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13491             as_Register($src2$$reg), ext::uxth);
13492    %}
13493   ins_pipe(ialu_reg_reg);
13494 %}
13495 
13496 // This pattern is automatically generated from aarch64_ad.m4
13497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13498 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13499 %{
13500   match(Set dst (SubL src1 (AndL src2 mask)));
13501   ins_cost(INSN_COST);
13502   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13503 
13504    ins_encode %{
13505      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13506             as_Register($src2$$reg), ext::uxtw);
13507    %}
13508   ins_pipe(ialu_reg_reg);
13509 %}
13510 
13511 
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13515 %{
13516   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13517   ins_cost(1.9 * INSN_COST);
13518   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13519 
13520    ins_encode %{
13521      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13522             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13523    %}
13524   ins_pipe(ialu_reg_reg_shift);
13525 %}
13526 
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13530 %{
13531   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13532   ins_cost(1.9 * INSN_COST);
13533   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13534 
13535    ins_encode %{
13536      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13537             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13538    %}
13539   ins_pipe(ialu_reg_reg_shift);
13540 %}
13541 
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13545 %{
13546   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13547   ins_cost(1.9 * INSN_COST);
13548   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13549 
13550    ins_encode %{
13551      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13552             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13553    %}
13554   ins_pipe(ialu_reg_reg_shift);
13555 %}
13556 
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13560 %{
13561   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13562   ins_cost(1.9 * INSN_COST);
13563   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13564 
13565    ins_encode %{
13566      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13567             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13568    %}
13569   ins_pipe(ialu_reg_reg_shift);
13570 %}
13571 
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13575 %{
13576   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13577   ins_cost(1.9 * INSN_COST);
13578   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13579 
13580    ins_encode %{
13581      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13582             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13583    %}
13584   ins_pipe(ialu_reg_reg_shift);
13585 %}
13586 
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13590 %{
13591   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13592   ins_cost(1.9 * INSN_COST);
13593   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13594 
13595    ins_encode %{
13596      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13597             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13598    %}
13599   ins_pipe(ialu_reg_reg_shift);
13600 %}
13601 
13602 // This pattern is automatically generated from aarch64_ad.m4
13603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13604 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13605 %{
13606   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13607   ins_cost(1.9 * INSN_COST);
13608   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13609 
13610    ins_encode %{
13611      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13612             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13613    %}
13614   ins_pipe(ialu_reg_reg_shift);
13615 %}
13616 
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13620 %{
13621   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13622   ins_cost(1.9 * INSN_COST);
13623   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13624 
13625    ins_encode %{
13626      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13627             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13628    %}
13629   ins_pipe(ialu_reg_reg_shift);
13630 %}
13631 
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13635 %{
13636   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13637   ins_cost(1.9 * INSN_COST);
13638   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13639 
13640    ins_encode %{
13641      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13642             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13643    %}
13644   ins_pipe(ialu_reg_reg_shift);
13645 %}
13646 
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13650 %{
13651   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13652   ins_cost(1.9 * INSN_COST);
13653   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13654 
13655    ins_encode %{
13656      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13657             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13658    %}
13659   ins_pipe(ialu_reg_reg_shift);
13660 %}
13661 
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13665 %{
13666   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13667   ins_cost(1.9 * INSN_COST);
13668   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13669 
13670    ins_encode %{
13671      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13672             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13673    %}
13674   ins_pipe(ialu_reg_reg_shift);
13675 %}
13676 
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13680 %{
13681   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13682   ins_cost(1.9 * INSN_COST);
13683   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13684 
13685    ins_encode %{
13686      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13687             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13688    %}
13689   ins_pipe(ialu_reg_reg_shift);
13690 %}
13691 
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13695 %{
13696   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13697   ins_cost(1.9 * INSN_COST);
13698   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13699 
13700    ins_encode %{
13701      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13702             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13703    %}
13704   ins_pipe(ialu_reg_reg_shift);
13705 %}
13706 
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13710 %{
13711   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13712   ins_cost(1.9 * INSN_COST);
13713   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13714 
13715    ins_encode %{
13716      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13717             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13718    %}
13719   ins_pipe(ialu_reg_reg_shift);
13720 %}
13721 
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13725 %{
13726   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13727   ins_cost(1.9 * INSN_COST);
13728   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13729 
13730    ins_encode %{
13731      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13732             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13733    %}
13734   ins_pipe(ialu_reg_reg_shift);
13735 %}
13736 
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13740 %{
13741   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13742   ins_cost(1.9 * INSN_COST);
13743   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13744 
13745    ins_encode %{
13746      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13747             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13748    %}
13749   ins_pipe(ialu_reg_reg_shift);
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13755 %{
13756   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13757   ins_cost(1.9 * INSN_COST);
13758   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13759 
13760    ins_encode %{
13761      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13762             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13763    %}
13764   ins_pipe(ialu_reg_reg_shift);
13765 %}
13766 
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13770 %{
13771   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13772   ins_cost(1.9 * INSN_COST);
13773   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13774 
13775    ins_encode %{
13776      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13777             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13778    %}
13779   ins_pipe(ialu_reg_reg_shift);
13780 %}
13781 
13782 // This pattern is automatically generated from aarch64_ad.m4
13783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13784 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13785 %{
13786   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13787   ins_cost(1.9 * INSN_COST);
13788   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13789 
13790    ins_encode %{
13791      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13792             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13793    %}
13794   ins_pipe(ialu_reg_reg_shift);
13795 %}
13796 
13797 // This pattern is automatically generated from aarch64_ad.m4
13798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13799 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13800 %{
13801   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13802   ins_cost(1.9 * INSN_COST);
13803   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13804 
13805    ins_encode %{
13806      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13807             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13808    %}
13809   ins_pipe(ialu_reg_reg_shift);
13810 %}
13811 
13812 // This pattern is automatically generated from aarch64_ad.m4
13813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13814 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13815 %{
13816   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13817   ins_cost(1.9 * INSN_COST);
13818   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13819 
13820    ins_encode %{
13821      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13822             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13823    %}
13824   ins_pipe(ialu_reg_reg_shift);
13825 %}
13826 
13827 // This pattern is automatically generated from aarch64_ad.m4
13828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13829 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13830 %{
13831   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13832   ins_cost(1.9 * INSN_COST);
13833   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13834 
13835    ins_encode %{
13836      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13837             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13838    %}
13839   ins_pipe(ialu_reg_reg_shift);
13840 %}
13841 
13842 
13843 
13844 // END This section of the file is automatically generated. Do not edit --------------
13845 
13846 
13847 // ============================================================================
13848 // Floating Point Arithmetic Instructions
13849 
13850 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13851   match(Set dst (AddF src1 src2));
13852 
13853   ins_cost(INSN_COST * 5);
13854   format %{ "fadds   $dst, $src1, $src2" %}
13855 
13856   ins_encode %{
13857     __ fadds(as_FloatRegister($dst$$reg),
13858              as_FloatRegister($src1$$reg),
13859              as_FloatRegister($src2$$reg));
13860   %}
13861 
13862   ins_pipe(fp_dop_reg_reg_s);
13863 %}
13864 
13865 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13866   match(Set dst (AddD src1 src2));
13867 
13868   ins_cost(INSN_COST * 5);
13869   format %{ "faddd   $dst, $src1, $src2" %}
13870 
13871   ins_encode %{
13872     __ faddd(as_FloatRegister($dst$$reg),
13873              as_FloatRegister($src1$$reg),
13874              as_FloatRegister($src2$$reg));
13875   %}
13876 
13877   ins_pipe(fp_dop_reg_reg_d);
13878 %}
13879 
13880 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13881   match(Set dst (SubF src1 src2));
13882 
13883   ins_cost(INSN_COST * 5);
13884   format %{ "fsubs   $dst, $src1, $src2" %}
13885 
13886   ins_encode %{
13887     __ fsubs(as_FloatRegister($dst$$reg),
13888              as_FloatRegister($src1$$reg),
13889              as_FloatRegister($src2$$reg));
13890   %}
13891 
13892   ins_pipe(fp_dop_reg_reg_s);
13893 %}
13894 
13895 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13896   match(Set dst (SubD src1 src2));
13897 
13898   ins_cost(INSN_COST * 5);
13899   format %{ "fsubd   $dst, $src1, $src2" %}
13900 
13901   ins_encode %{
13902     __ fsubd(as_FloatRegister($dst$$reg),
13903              as_FloatRegister($src1$$reg),
13904              as_FloatRegister($src2$$reg));
13905   %}
13906 
13907   ins_pipe(fp_dop_reg_reg_d);
13908 %}
13909 
13910 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13911   match(Set dst (MulF src1 src2));
13912 
13913   ins_cost(INSN_COST * 6);
13914   format %{ "fmuls   $dst, $src1, $src2" %}
13915 
13916   ins_encode %{
13917     __ fmuls(as_FloatRegister($dst$$reg),
13918              as_FloatRegister($src1$$reg),
13919              as_FloatRegister($src2$$reg));
13920   %}
13921 
13922   ins_pipe(fp_dop_reg_reg_s);
13923 %}
13924 
13925 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13926   match(Set dst (MulD src1 src2));
13927 
13928   ins_cost(INSN_COST * 6);
13929   format %{ "fmuld   $dst, $src1, $src2" %}
13930 
13931   ins_encode %{
13932     __ fmuld(as_FloatRegister($dst$$reg),
13933              as_FloatRegister($src1$$reg),
13934              as_FloatRegister($src2$$reg));
13935   %}
13936 
13937   ins_pipe(fp_dop_reg_reg_d);
13938 %}
13939 
13940 // src1 * src2 + src3
13941 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13942   predicate(UseFMA);
13943   match(Set dst (FmaF src3 (Binary src1 src2)));
13944 
13945   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13946 
13947   ins_encode %{
13948     __ fmadds(as_FloatRegister($dst$$reg),
13949              as_FloatRegister($src1$$reg),
13950              as_FloatRegister($src2$$reg),
13951              as_FloatRegister($src3$$reg));
13952   %}
13953 
13954   ins_pipe(pipe_class_default);
13955 %}
13956 
13957 // src1 * src2 + src3
13958 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13959   predicate(UseFMA);
13960   match(Set dst (FmaD src3 (Binary src1 src2)));
13961 
13962   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13963 
13964   ins_encode %{
13965     __ fmaddd(as_FloatRegister($dst$$reg),
13966              as_FloatRegister($src1$$reg),
13967              as_FloatRegister($src2$$reg),
13968              as_FloatRegister($src3$$reg));
13969   %}
13970 
13971   ins_pipe(pipe_class_default);
13972 %}
13973 
13974 // -src1 * src2 + src3
13975 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13976   predicate(UseFMA);
13977   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13978   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13979 
13980   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13981 
13982   ins_encode %{
13983     __ fmsubs(as_FloatRegister($dst$$reg),
13984               as_FloatRegister($src1$$reg),
13985               as_FloatRegister($src2$$reg),
13986               as_FloatRegister($src3$$reg));
13987   %}
13988 
13989   ins_pipe(pipe_class_default);
13990 %}
13991 
13992 // -src1 * src2 + src3
13993 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13994   predicate(UseFMA);
13995   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13996   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13997 
13998   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13999 
14000   ins_encode %{
14001     __ fmsubd(as_FloatRegister($dst$$reg),
14002               as_FloatRegister($src1$$reg),
14003               as_FloatRegister($src2$$reg),
14004               as_FloatRegister($src3$$reg));
14005   %}
14006 
14007   ins_pipe(pipe_class_default);
14008 %}
14009 
14010 // -src1 * src2 - src3
14011 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14012   predicate(UseFMA);
14013   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14014   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14015 
14016   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14017 
14018   ins_encode %{
14019     __ fnmadds(as_FloatRegister($dst$$reg),
14020                as_FloatRegister($src1$$reg),
14021                as_FloatRegister($src2$$reg),
14022                as_FloatRegister($src3$$reg));
14023   %}
14024 
14025   ins_pipe(pipe_class_default);
14026 %}
14027 
14028 // -src1 * src2 - src3
14029 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14030   predicate(UseFMA);
14031   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14032   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14033 
14034   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14035 
14036   ins_encode %{
14037     __ fnmaddd(as_FloatRegister($dst$$reg),
14038                as_FloatRegister($src1$$reg),
14039                as_FloatRegister($src2$$reg),
14040                as_FloatRegister($src3$$reg));
14041   %}
14042 
14043   ins_pipe(pipe_class_default);
14044 %}
14045 
14046 // src1 * src2 - src3
14047 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14048   predicate(UseFMA);
14049   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14050 
14051   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14052 
14053   ins_encode %{
14054     __ fnmsubs(as_FloatRegister($dst$$reg),
14055                as_FloatRegister($src1$$reg),
14056                as_FloatRegister($src2$$reg),
14057                as_FloatRegister($src3$$reg));
14058   %}
14059 
14060   ins_pipe(pipe_class_default);
14061 %}
14062 
14063 // src1 * src2 - src3
14064 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14065   predicate(UseFMA);
14066   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14067 
14068   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14069 
14070   ins_encode %{
14071   // n.b. insn name should be fnmsubd
14072     __ fnmsub(as_FloatRegister($dst$$reg),
14073               as_FloatRegister($src1$$reg),
14074               as_FloatRegister($src2$$reg),
14075               as_FloatRegister($src3$$reg));
14076   %}
14077 
14078   ins_pipe(pipe_class_default);
14079 %}
14080 
14081 
14082 // Math.max(FF)F
14083 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14084   match(Set dst (MaxF src1 src2));
14085 
14086   format %{ "fmaxs   $dst, $src1, $src2" %}
14087   ins_encode %{
14088     __ fmaxs(as_FloatRegister($dst$$reg),
14089              as_FloatRegister($src1$$reg),
14090              as_FloatRegister($src2$$reg));
14091   %}
14092 
14093   ins_pipe(fp_dop_reg_reg_s);
14094 %}
14095 
14096 // Math.min(FF)F
14097 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14098   match(Set dst (MinF src1 src2));
14099 
14100   format %{ "fmins   $dst, $src1, $src2" %}
14101   ins_encode %{
14102     __ fmins(as_FloatRegister($dst$$reg),
14103              as_FloatRegister($src1$$reg),
14104              as_FloatRegister($src2$$reg));
14105   %}
14106 
14107   ins_pipe(fp_dop_reg_reg_s);
14108 %}
14109 
14110 // Math.max(DD)D
14111 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14112   match(Set dst (MaxD src1 src2));
14113 
14114   format %{ "fmaxd   $dst, $src1, $src2" %}
14115   ins_encode %{
14116     __ fmaxd(as_FloatRegister($dst$$reg),
14117              as_FloatRegister($src1$$reg),
14118              as_FloatRegister($src2$$reg));
14119   %}
14120 
14121   ins_pipe(fp_dop_reg_reg_d);
14122 %}
14123 
14124 // Math.min(DD)D
14125 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14126   match(Set dst (MinD src1 src2));
14127 
14128   format %{ "fmind   $dst, $src1, $src2" %}
14129   ins_encode %{
14130     __ fmind(as_FloatRegister($dst$$reg),
14131              as_FloatRegister($src1$$reg),
14132              as_FloatRegister($src2$$reg));
14133   %}
14134 
14135   ins_pipe(fp_dop_reg_reg_d);
14136 %}
14137 
14138 
14139 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14140   match(Set dst (DivF src1  src2));
14141 
14142   ins_cost(INSN_COST * 18);
14143   format %{ "fdivs   $dst, $src1, $src2" %}
14144 
14145   ins_encode %{
14146     __ fdivs(as_FloatRegister($dst$$reg),
14147              as_FloatRegister($src1$$reg),
14148              as_FloatRegister($src2$$reg));
14149   %}
14150 
14151   ins_pipe(fp_div_s);
14152 %}
14153 
14154 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14155   match(Set dst (DivD src1  src2));
14156 
14157   ins_cost(INSN_COST * 32);
14158   format %{ "fdivd   $dst, $src1, $src2" %}
14159 
14160   ins_encode %{
14161     __ fdivd(as_FloatRegister($dst$$reg),
14162              as_FloatRegister($src1$$reg),
14163              as_FloatRegister($src2$$reg));
14164   %}
14165 
14166   ins_pipe(fp_div_d);
14167 %}
14168 
14169 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14170   match(Set dst (NegF src));
14171 
14172   ins_cost(INSN_COST * 3);
14173   format %{ "fneg   $dst, $src" %}
14174 
14175   ins_encode %{
14176     __ fnegs(as_FloatRegister($dst$$reg),
14177              as_FloatRegister($src$$reg));
14178   %}
14179 
14180   ins_pipe(fp_uop_s);
14181 %}
14182 
14183 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14184   match(Set dst (NegD src));
14185 
14186   ins_cost(INSN_COST * 3);
14187   format %{ "fnegd   $dst, $src" %}
14188 
14189   ins_encode %{
14190     __ fnegd(as_FloatRegister($dst$$reg),
14191              as_FloatRegister($src$$reg));
14192   %}
14193 
14194   ins_pipe(fp_uop_d);
14195 %}
14196 
14197 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14198 %{
14199   match(Set dst (AbsI src));
14200 
14201   effect(KILL cr);
14202   ins_cost(INSN_COST * 2);
14203   format %{ "cmpw  $src, zr\n\t"
14204             "cnegw $dst, $src, Assembler::LT\t# int abs"
14205   %}
14206 
14207   ins_encode %{
14208     __ cmpw(as_Register($src$$reg), zr);
14209     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14210   %}
14211   ins_pipe(pipe_class_default);
14212 %}
14213 
14214 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14215 %{
14216   match(Set dst (AbsL src));
14217 
14218   effect(KILL cr);
14219   ins_cost(INSN_COST * 2);
14220   format %{ "cmp  $src, zr\n\t"
14221             "cneg $dst, $src, Assembler::LT\t# long abs"
14222   %}
14223 
14224   ins_encode %{
14225     __ cmp(as_Register($src$$reg), zr);
14226     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14227   %}
14228   ins_pipe(pipe_class_default);
14229 %}
14230 
14231 instruct absF_reg(vRegF dst, vRegF src) %{
14232   match(Set dst (AbsF src));
14233 
14234   ins_cost(INSN_COST * 3);
14235   format %{ "fabss   $dst, $src" %}
14236   ins_encode %{
14237     __ fabss(as_FloatRegister($dst$$reg),
14238              as_FloatRegister($src$$reg));
14239   %}
14240 
14241   ins_pipe(fp_uop_s);
14242 %}
14243 
14244 instruct absD_reg(vRegD dst, vRegD src) %{
14245   match(Set dst (AbsD src));
14246 
14247   ins_cost(INSN_COST * 3);
14248   format %{ "fabsd   $dst, $src" %}
14249   ins_encode %{
14250     __ fabsd(as_FloatRegister($dst$$reg),
14251              as_FloatRegister($src$$reg));
14252   %}
14253 
14254   ins_pipe(fp_uop_d);
14255 %}
14256 
14257 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14258   match(Set dst (AbsF (SubF src1 src2)));
14259 
14260   ins_cost(INSN_COST * 3);
14261   format %{ "fabds   $dst, $src1, $src2" %}
14262   ins_encode %{
14263     __ fabds(as_FloatRegister($dst$$reg),
14264              as_FloatRegister($src1$$reg),
14265              as_FloatRegister($src2$$reg));
14266   %}
14267 
14268   ins_pipe(fp_uop_s);
14269 %}
14270 
14271 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14272   match(Set dst (AbsD (SubD src1 src2)));
14273 
14274   ins_cost(INSN_COST * 3);
14275   format %{ "fabdd   $dst, $src1, $src2" %}
14276   ins_encode %{
14277     __ fabdd(as_FloatRegister($dst$$reg),
14278              as_FloatRegister($src1$$reg),
14279              as_FloatRegister($src2$$reg));
14280   %}
14281 
14282   ins_pipe(fp_uop_d);
14283 %}
14284 
14285 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14286   match(Set dst (SqrtD src));
14287 
14288   ins_cost(INSN_COST * 50);
14289   format %{ "fsqrtd  $dst, $src" %}
14290   ins_encode %{
14291     __ fsqrtd(as_FloatRegister($dst$$reg),
14292              as_FloatRegister($src$$reg));
14293   %}
14294 
14295   ins_pipe(fp_div_s);
14296 %}
14297 
14298 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14299   match(Set dst (SqrtF src));
14300 
14301   ins_cost(INSN_COST * 50);
14302   format %{ "fsqrts  $dst, $src" %}
14303   ins_encode %{
14304     __ fsqrts(as_FloatRegister($dst$$reg),
14305              as_FloatRegister($src$$reg));
14306   %}
14307 
14308   ins_pipe(fp_div_d);
14309 %}
14310 
14311 // Math.rint, floor, ceil
14312 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14313   match(Set dst (RoundDoubleMode src rmode));
14314   format %{ "frint  $dst, $src, $rmode" %}
14315   ins_encode %{
14316     switch ($rmode$$constant) {
14317       case RoundDoubleModeNode::rmode_rint:
14318         __ frintnd(as_FloatRegister($dst$$reg),
14319                    as_FloatRegister($src$$reg));
14320         break;
14321       case RoundDoubleModeNode::rmode_floor:
14322         __ frintmd(as_FloatRegister($dst$$reg),
14323                    as_FloatRegister($src$$reg));
14324         break;
14325       case RoundDoubleModeNode::rmode_ceil:
14326         __ frintpd(as_FloatRegister($dst$$reg),
14327                    as_FloatRegister($src$$reg));
14328         break;
14329     }
14330   %}
14331   ins_pipe(fp_uop_d);
14332 %}
14333 
14334 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14335   match(Set dst (CopySignD src1 (Binary src2 zero)));
14336   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14337   format %{ "CopySignD  $dst $src1 $src2" %}
14338   ins_encode %{
14339     FloatRegister dst = as_FloatRegister($dst$$reg),
14340                   src1 = as_FloatRegister($src1$$reg),
14341                   src2 = as_FloatRegister($src2$$reg),
14342                   zero = as_FloatRegister($zero$$reg);
14343     __ fnegd(dst, zero);
14344     __ bsl(dst, __ T8B, src2, src1);
14345   %}
14346   ins_pipe(fp_uop_d);
14347 %}
14348 
14349 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14350   match(Set dst (CopySignF src1 src2));
14351   effect(TEMP_DEF dst, USE src1, USE src2);
14352   format %{ "CopySignF  $dst $src1 $src2" %}
14353   ins_encode %{
14354     FloatRegister dst = as_FloatRegister($dst$$reg),
14355                   src1 = as_FloatRegister($src1$$reg),
14356                   src2 = as_FloatRegister($src2$$reg);
14357     __ movi(dst, __ T2S, 0x80, 24);
14358     __ bsl(dst, __ T8B, src2, src1);
14359   %}
14360   ins_pipe(fp_uop_d);
14361 %}
14362 
14363 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14364   match(Set dst (SignumD src (Binary zero one)));
14365   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14366   format %{ "signumD  $dst, $src" %}
14367   ins_encode %{
14368     FloatRegister src = as_FloatRegister($src$$reg),
14369                   dst = as_FloatRegister($dst$$reg),
14370                   zero = as_FloatRegister($zero$$reg),
14371                   one = as_FloatRegister($one$$reg);
14372     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14373     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14374     // Bit selection instruction gets bit from "one" for each enabled bit in
14375     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14376     // NaN the whole "src" will be copied because "dst" is zero. For all other
14377     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14378     // from "src", and all other bits are copied from 1.0.
14379     __ bsl(dst, __ T8B, one, src);
14380   %}
14381   ins_pipe(fp_uop_d);
14382 %}
14383 
14384 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14385   match(Set dst (SignumF src (Binary zero one)));
14386   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14387   format %{ "signumF  $dst, $src" %}
14388   ins_encode %{
14389     FloatRegister src = as_FloatRegister($src$$reg),
14390                   dst = as_FloatRegister($dst$$reg),
14391                   zero = as_FloatRegister($zero$$reg),
14392                   one = as_FloatRegister($one$$reg);
14393     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14394     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14395     // Bit selection instruction gets bit from "one" for each enabled bit in
14396     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14397     // NaN the whole "src" will be copied because "dst" is zero. For all other
14398     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14399     // from "src", and all other bits are copied from 1.0.
14400     __ bsl(dst, __ T8B, one, src);
14401   %}
14402   ins_pipe(fp_uop_d);
14403 %}
14404 
14405 // ============================================================================
14406 // Logical Instructions
14407 
14408 // Integer Logical Instructions
14409 
14410 // And Instructions
14411 
14412 
14413 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14414   match(Set dst (AndI src1 src2));
14415 
14416   format %{ "andw  $dst, $src1, $src2\t# int" %}
14417 
14418   ins_cost(INSN_COST);
14419   ins_encode %{
14420     __ andw(as_Register($dst$$reg),
14421             as_Register($src1$$reg),
14422             as_Register($src2$$reg));
14423   %}
14424 
14425   ins_pipe(ialu_reg_reg);
14426 %}
14427 
14428 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14429   match(Set dst (AndI src1 src2));
14430 
14431   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14432 
14433   ins_cost(INSN_COST);
14434   ins_encode %{
14435     __ andw(as_Register($dst$$reg),
14436             as_Register($src1$$reg),
14437             (uint64_t)($src2$$constant));
14438   %}
14439 
14440   ins_pipe(ialu_reg_imm);
14441 %}
14442 
14443 // Or Instructions
14444 
14445 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14446   match(Set dst (OrI src1 src2));
14447 
14448   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14449 
14450   ins_cost(INSN_COST);
14451   ins_encode %{
14452     __ orrw(as_Register($dst$$reg),
14453             as_Register($src1$$reg),
14454             as_Register($src2$$reg));
14455   %}
14456 
14457   ins_pipe(ialu_reg_reg);
14458 %}
14459 
14460 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14461   match(Set dst (OrI src1 src2));
14462 
14463   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14464 
14465   ins_cost(INSN_COST);
14466   ins_encode %{
14467     __ orrw(as_Register($dst$$reg),
14468             as_Register($src1$$reg),
14469             (uint64_t)($src2$$constant));
14470   %}
14471 
14472   ins_pipe(ialu_reg_imm);
14473 %}
14474 
14475 // Xor Instructions
14476 
14477 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14478   match(Set dst (XorI src1 src2));
14479 
14480   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14481 
14482   ins_cost(INSN_COST);
14483   ins_encode %{
14484     __ eorw(as_Register($dst$$reg),
14485             as_Register($src1$$reg),
14486             as_Register($src2$$reg));
14487   %}
14488 
14489   ins_pipe(ialu_reg_reg);
14490 %}
14491 
14492 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14493   match(Set dst (XorI src1 src2));
14494 
14495   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14496 
14497   ins_cost(INSN_COST);
14498   ins_encode %{
14499     __ eorw(as_Register($dst$$reg),
14500             as_Register($src1$$reg),
14501             (uint64_t)($src2$$constant));
14502   %}
14503 
14504   ins_pipe(ialu_reg_imm);
14505 %}
14506 
14507 // Long Logical Instructions
14508 // TODO
14509 
14510 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14511   match(Set dst (AndL src1 src2));
14512 
14513   format %{ "and  $dst, $src1, $src2\t# int" %}
14514 
14515   ins_cost(INSN_COST);
14516   ins_encode %{
14517     __ andr(as_Register($dst$$reg),
14518             as_Register($src1$$reg),
14519             as_Register($src2$$reg));
14520   %}
14521 
14522   ins_pipe(ialu_reg_reg);
14523 %}
14524 
14525 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14526   match(Set dst (AndL src1 src2));
14527 
14528   format %{ "and  $dst, $src1, $src2\t# int" %}
14529 
14530   ins_cost(INSN_COST);
14531   ins_encode %{
14532     __ andr(as_Register($dst$$reg),
14533             as_Register($src1$$reg),
14534             (uint64_t)($src2$$constant));
14535   %}
14536 
14537   ins_pipe(ialu_reg_imm);
14538 %}
14539 
14540 // Or Instructions
14541 
14542 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14543   match(Set dst (OrL src1 src2));
14544 
14545   format %{ "orr  $dst, $src1, $src2\t# int" %}
14546 
14547   ins_cost(INSN_COST);
14548   ins_encode %{
14549     __ orr(as_Register($dst$$reg),
14550            as_Register($src1$$reg),
14551            as_Register($src2$$reg));
14552   %}
14553 
14554   ins_pipe(ialu_reg_reg);
14555 %}
14556 
14557 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14558   match(Set dst (OrL src1 src2));
14559 
14560   format %{ "orr  $dst, $src1, $src2\t# int" %}
14561 
14562   ins_cost(INSN_COST);
14563   ins_encode %{
14564     __ orr(as_Register($dst$$reg),
14565            as_Register($src1$$reg),
14566            (uint64_t)($src2$$constant));
14567   %}
14568 
14569   ins_pipe(ialu_reg_imm);
14570 %}
14571 
14572 // Xor Instructions
14573 
14574 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14575   match(Set dst (XorL src1 src2));
14576 
14577   format %{ "eor  $dst, $src1, $src2\t# int" %}
14578 
14579   ins_cost(INSN_COST);
14580   ins_encode %{
14581     __ eor(as_Register($dst$$reg),
14582            as_Register($src1$$reg),
14583            as_Register($src2$$reg));
14584   %}
14585 
14586   ins_pipe(ialu_reg_reg);
14587 %}
14588 
14589 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14590   match(Set dst (XorL src1 src2));
14591 
14592   ins_cost(INSN_COST);
14593   format %{ "eor  $dst, $src1, $src2\t# int" %}
14594 
14595   ins_encode %{
14596     __ eor(as_Register($dst$$reg),
14597            as_Register($src1$$reg),
14598            (uint64_t)($src2$$constant));
14599   %}
14600 
14601   ins_pipe(ialu_reg_imm);
14602 %}
14603 
14604 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14605 %{
14606   match(Set dst (ConvI2L src));
14607 
14608   ins_cost(INSN_COST);
14609   format %{ "sxtw  $dst, $src\t# i2l" %}
14610   ins_encode %{
14611     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14612   %}
14613   ins_pipe(ialu_reg_shift);
14614 %}
14615 
14616 // this pattern occurs in bigmath arithmetic
14617 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14618 %{
14619   match(Set dst (AndL (ConvI2L src) mask));
14620 
14621   ins_cost(INSN_COST);
14622   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14623   ins_encode %{
14624     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14625   %}
14626 
14627   ins_pipe(ialu_reg_shift);
14628 %}
14629 
14630 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14631   match(Set dst (ConvL2I src));
14632 
14633   ins_cost(INSN_COST);
14634   format %{ "movw  $dst, $src \t// l2i" %}
14635 
14636   ins_encode %{
14637     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14638   %}
14639 
14640   ins_pipe(ialu_reg);
14641 %}
14642 
14643 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14644 %{
14645   match(Set dst (Conv2B src));
14646   effect(KILL cr);
14647 
14648   format %{
14649     "cmpw $src, zr\n\t"
14650     "cset $dst, ne"
14651   %}
14652 
14653   ins_encode %{
14654     __ cmpw(as_Register($src$$reg), zr);
14655     __ cset(as_Register($dst$$reg), Assembler::NE);
14656   %}
14657 
14658   ins_pipe(ialu_reg);
14659 %}
14660 
14661 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14662 %{
14663   match(Set dst (Conv2B src));
14664   effect(KILL cr);
14665 
14666   format %{
14667     "cmp  $src, zr\n\t"
14668     "cset $dst, ne"
14669   %}
14670 
14671   ins_encode %{
14672     __ cmp(as_Register($src$$reg), zr);
14673     __ cset(as_Register($dst$$reg), Assembler::NE);
14674   %}
14675 
14676   ins_pipe(ialu_reg);
14677 %}
14678 
14679 instruct convD2F_reg(vRegF dst, vRegD src) %{
14680   match(Set dst (ConvD2F src));
14681 
14682   ins_cost(INSN_COST * 5);
14683   format %{ "fcvtd  $dst, $src \t// d2f" %}
14684 
14685   ins_encode %{
14686     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14687   %}
14688 
14689   ins_pipe(fp_d2f);
14690 %}
14691 
14692 instruct convF2D_reg(vRegD dst, vRegF src) %{
14693   match(Set dst (ConvF2D src));
14694 
14695   ins_cost(INSN_COST * 5);
14696   format %{ "fcvts  $dst, $src \t// f2d" %}
14697 
14698   ins_encode %{
14699     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14700   %}
14701 
14702   ins_pipe(fp_f2d);
14703 %}
14704 
14705 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14706   match(Set dst (ConvF2I src));
14707 
14708   ins_cost(INSN_COST * 5);
14709   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14710 
14711   ins_encode %{
14712     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14713   %}
14714 
14715   ins_pipe(fp_f2i);
14716 %}
14717 
14718 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14719   match(Set dst (ConvF2L src));
14720 
14721   ins_cost(INSN_COST * 5);
14722   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14723 
14724   ins_encode %{
14725     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14726   %}
14727 
14728   ins_pipe(fp_f2l);
14729 %}
14730 
14731 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14732   match(Set dst (ConvI2F src));
14733 
14734   ins_cost(INSN_COST * 5);
14735   format %{ "scvtfws  $dst, $src \t// i2f" %}
14736 
14737   ins_encode %{
14738     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14739   %}
14740 
14741   ins_pipe(fp_i2f);
14742 %}
14743 
14744 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14745   match(Set dst (ConvL2F src));
14746 
14747   ins_cost(INSN_COST * 5);
14748   format %{ "scvtfs  $dst, $src \t// l2f" %}
14749 
14750   ins_encode %{
14751     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14752   %}
14753 
14754   ins_pipe(fp_l2f);
14755 %}
14756 
14757 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14758   match(Set dst (ConvD2I src));
14759 
14760   ins_cost(INSN_COST * 5);
14761   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14762 
14763   ins_encode %{
14764     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14765   %}
14766 
14767   ins_pipe(fp_d2i);
14768 %}
14769 
14770 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14771   match(Set dst (ConvD2L src));
14772 
14773   ins_cost(INSN_COST * 5);
14774   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14775 
14776   ins_encode %{
14777     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14778   %}
14779 
14780   ins_pipe(fp_d2l);
14781 %}
14782 
14783 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14784   match(Set dst (ConvI2D src));
14785 
14786   ins_cost(INSN_COST * 5);
14787   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14788 
14789   ins_encode %{
14790     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14791   %}
14792 
14793   ins_pipe(fp_i2d);
14794 %}
14795 
14796 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14797   match(Set dst (ConvL2D src));
14798 
14799   ins_cost(INSN_COST * 5);
14800   format %{ "scvtfd  $dst, $src \t// l2d" %}
14801 
14802   ins_encode %{
14803     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14804   %}
14805 
14806   ins_pipe(fp_l2d);
14807 %}
14808 
14809 // stack <-> reg and reg <-> reg shuffles with no conversion
14810 
14811 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14812 
14813   match(Set dst (MoveF2I src));
14814 
14815   effect(DEF dst, USE src);
14816 
14817   ins_cost(4 * INSN_COST);
14818 
14819   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14820 
14821   ins_encode %{
14822     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14823   %}
14824 
14825   ins_pipe(iload_reg_reg);
14826 
14827 %}
14828 
14829 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14830 
14831   match(Set dst (MoveI2F src));
14832 
14833   effect(DEF dst, USE src);
14834 
14835   ins_cost(4 * INSN_COST);
14836 
14837   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14838 
14839   ins_encode %{
14840     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14841   %}
14842 
14843   ins_pipe(pipe_class_memory);
14844 
14845 %}
14846 
14847 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14848 
14849   match(Set dst (MoveD2L src));
14850 
14851   effect(DEF dst, USE src);
14852 
14853   ins_cost(4 * INSN_COST);
14854 
14855   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14856 
14857   ins_encode %{
14858     __ ldr($dst$$Register, Address(sp, $src$$disp));
14859   %}
14860 
14861   ins_pipe(iload_reg_reg);
14862 
14863 %}
14864 
14865 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14866 
14867   match(Set dst (MoveL2D src));
14868 
14869   effect(DEF dst, USE src);
14870 
14871   ins_cost(4 * INSN_COST);
14872 
14873   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14874 
14875   ins_encode %{
14876     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14877   %}
14878 
14879   ins_pipe(pipe_class_memory);
14880 
14881 %}
14882 
14883 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14884 
14885   match(Set dst (MoveF2I src));
14886 
14887   effect(DEF dst, USE src);
14888 
14889   ins_cost(INSN_COST);
14890 
14891   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14892 
14893   ins_encode %{
14894     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14895   %}
14896 
14897   ins_pipe(pipe_class_memory);
14898 
14899 %}
14900 
14901 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14902 
14903   match(Set dst (MoveI2F src));
14904 
14905   effect(DEF dst, USE src);
14906 
14907   ins_cost(INSN_COST);
14908 
14909   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14910 
14911   ins_encode %{
14912     __ strw($src$$Register, Address(sp, $dst$$disp));
14913   %}
14914 
14915   ins_pipe(istore_reg_reg);
14916 
14917 %}
14918 
14919 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14920 
14921   match(Set dst (MoveD2L src));
14922 
14923   effect(DEF dst, USE src);
14924 
14925   ins_cost(INSN_COST);
14926 
14927   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14928 
14929   ins_encode %{
14930     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14931   %}
14932 
14933   ins_pipe(pipe_class_memory);
14934 
14935 %}
14936 
14937 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14938 
14939   match(Set dst (MoveL2D src));
14940 
14941   effect(DEF dst, USE src);
14942 
14943   ins_cost(INSN_COST);
14944 
14945   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14946 
14947   ins_encode %{
14948     __ str($src$$Register, Address(sp, $dst$$disp));
14949   %}
14950 
14951   ins_pipe(istore_reg_reg);
14952 
14953 %}
14954 
14955 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14956 
14957   match(Set dst (MoveF2I src));
14958 
14959   effect(DEF dst, USE src);
14960 
14961   ins_cost(INSN_COST);
14962 
14963   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14964 
14965   ins_encode %{
14966     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14967   %}
14968 
14969   ins_pipe(fp_f2i);
14970 
14971 %}
14972 
14973 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14974 
14975   match(Set dst (MoveI2F src));
14976 
14977   effect(DEF dst, USE src);
14978 
14979   ins_cost(INSN_COST);
14980 
14981   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14982 
14983   ins_encode %{
14984     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14985   %}
14986 
14987   ins_pipe(fp_i2f);
14988 
14989 %}
14990 
14991 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14992 
14993   match(Set dst (MoveD2L src));
14994 
14995   effect(DEF dst, USE src);
14996 
14997   ins_cost(INSN_COST);
14998 
14999   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15000 
15001   ins_encode %{
15002     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15003   %}
15004 
15005   ins_pipe(fp_d2l);
15006 
15007 %}
15008 
15009 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15010 
15011   match(Set dst (MoveL2D src));
15012 
15013   effect(DEF dst, USE src);
15014 
15015   ins_cost(INSN_COST);
15016 
15017   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15018 
15019   ins_encode %{
15020     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15021   %}
15022 
15023   ins_pipe(fp_l2d);
15024 
15025 %}
15026 
15027 // ============================================================================
15028 // clearing of an array
15029 
15030 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15031 %{
15032   match(Set dummy (ClearArray (Binary cnt base) zero));
15033   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15034 
15035   ins_cost(4 * INSN_COST);
15036   format %{ "ClearArray $cnt, $base" %}
15037 
15038   ins_encode %{
15039     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15040     if (tpc == NULL) {
15041       ciEnv::current()->record_failure("CodeCache is full");
15042       return;
15043     }
15044   %}
15045 
15046   ins_pipe(pipe_class_memory);
15047 %}
15048 
15049 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15050 %{
15051   predicate(((ClearArrayNode*)n)->word_copy_only());
15052   match(Set dummy (ClearArray (Binary cnt base) val));
15053   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15054 
15055   ins_cost(4 * INSN_COST);
15056   format %{ "ClearArray $cnt, $base, $val" %}
15057 
15058   ins_encode %{
15059     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15060   %}
15061 
15062   ins_pipe(pipe_class_memory);
15063 %}
15064 
15065 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15066 %{
15067   predicate((uint64_t)n->in(2)->get_long()
15068             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15069             && !((ClearArrayNode*)n)->word_copy_only());
15070   match(Set dummy (ClearArray cnt base));
15071   effect(TEMP temp, USE_KILL base, KILL cr);
15072 
15073   ins_cost(4 * INSN_COST);
15074   format %{ "ClearArray $cnt, $base" %}
15075 
15076   ins_encode %{
15077     __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15078   %}
15079 
15080   ins_pipe(pipe_class_memory);
15081 %}
15082 
15083 // ============================================================================
15084 // Overflow Math Instructions
15085 
15086 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15087 %{
15088   match(Set cr (OverflowAddI op1 op2));
15089 
15090   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15091   ins_cost(INSN_COST);
15092   ins_encode %{
15093     __ cmnw($op1$$Register, $op2$$Register);
15094   %}
15095 
15096   ins_pipe(icmp_reg_reg);
15097 %}
15098 
15099 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15100 %{
15101   match(Set cr (OverflowAddI op1 op2));
15102 
15103   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15104   ins_cost(INSN_COST);
15105   ins_encode %{
15106     __ cmnw($op1$$Register, $op2$$constant);
15107   %}
15108 
15109   ins_pipe(icmp_reg_imm);
15110 %}
15111 
15112 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15113 %{
15114   match(Set cr (OverflowAddL op1 op2));
15115 
15116   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15117   ins_cost(INSN_COST);
15118   ins_encode %{
15119     __ cmn($op1$$Register, $op2$$Register);
15120   %}
15121 
15122   ins_pipe(icmp_reg_reg);
15123 %}
15124 
15125 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15126 %{
15127   match(Set cr (OverflowAddL op1 op2));
15128 
15129   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15130   ins_cost(INSN_COST);
15131   ins_encode %{
15132     __ cmn($op1$$Register, $op2$$constant);
15133   %}
15134 
15135   ins_pipe(icmp_reg_imm);
15136 %}
15137 
15138 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15139 %{
15140   match(Set cr (OverflowSubI op1 op2));
15141 
15142   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15143   ins_cost(INSN_COST);
15144   ins_encode %{
15145     __ cmpw($op1$$Register, $op2$$Register);
15146   %}
15147 
15148   ins_pipe(icmp_reg_reg);
15149 %}
15150 
15151 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15152 %{
15153   match(Set cr (OverflowSubI op1 op2));
15154 
15155   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15156   ins_cost(INSN_COST);
15157   ins_encode %{
15158     __ cmpw($op1$$Register, $op2$$constant);
15159   %}
15160 
15161   ins_pipe(icmp_reg_imm);
15162 %}
15163 
15164 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15165 %{
15166   match(Set cr (OverflowSubL op1 op2));
15167 
15168   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15169   ins_cost(INSN_COST);
15170   ins_encode %{
15171     __ cmp($op1$$Register, $op2$$Register);
15172   %}
15173 
15174   ins_pipe(icmp_reg_reg);
15175 %}
15176 
15177 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15178 %{
15179   match(Set cr (OverflowSubL op1 op2));
15180 
15181   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15182   ins_cost(INSN_COST);
15183   ins_encode %{
15184     __ subs(zr, $op1$$Register, $op2$$constant);
15185   %}
15186 
15187   ins_pipe(icmp_reg_imm);
15188 %}
15189 
15190 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15191 %{
15192   match(Set cr (OverflowSubI zero op1));
15193 
15194   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15195   ins_cost(INSN_COST);
15196   ins_encode %{
15197     __ cmpw(zr, $op1$$Register);
15198   %}
15199 
15200   ins_pipe(icmp_reg_imm);
15201 %}
15202 
15203 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15204 %{
15205   match(Set cr (OverflowSubL zero op1));
15206 
15207   format %{ "cmp   zr, $op1\t# overflow check long" %}
15208   ins_cost(INSN_COST);
15209   ins_encode %{
15210     __ cmp(zr, $op1$$Register);
15211   %}
15212 
15213   ins_pipe(icmp_reg_imm);
15214 %}
15215 
15216 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15217 %{
15218   match(Set cr (OverflowMulI op1 op2));
15219 
15220   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15221             "cmp   rscratch1, rscratch1, sxtw\n\t"
15222             "movw  rscratch1, #0x80000000\n\t"
15223             "cselw rscratch1, rscratch1, zr, NE\n\t"
15224             "cmpw  rscratch1, #1" %}
15225   ins_cost(5 * INSN_COST);
15226   ins_encode %{
15227     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15228     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15229     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15230     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15231     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15232   %}
15233 
15234   ins_pipe(pipe_slow);
15235 %}
15236 
15237 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15238 %{
15239   match(If cmp (OverflowMulI op1 op2));
15240   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15241             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15242   effect(USE labl, KILL cr);
15243 
15244   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15245             "cmp   rscratch1, rscratch1, sxtw\n\t"
15246             "b$cmp   $labl" %}
15247   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15248   ins_encode %{
15249     Label* L = $labl$$label;
15250     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15251     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15252     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15253     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15254   %}
15255 
15256   ins_pipe(pipe_serial);
15257 %}
15258 
15259 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15260 %{
15261   match(Set cr (OverflowMulL op1 op2));
15262 
15263   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15264             "smulh rscratch2, $op1, $op2\n\t"
15265             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15266             "movw  rscratch1, #0x80000000\n\t"
15267             "cselw rscratch1, rscratch1, zr, NE\n\t"
15268             "cmpw  rscratch1, #1" %}
15269   ins_cost(6 * INSN_COST);
15270   ins_encode %{
15271     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15272     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15273     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15274     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15275     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15276     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15277   %}
15278 
15279   ins_pipe(pipe_slow);
15280 %}
15281 
15282 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15283 %{
15284   match(If cmp (OverflowMulL op1 op2));
15285   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15286             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15287   effect(USE labl, KILL cr);
15288 
15289   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15290             "smulh rscratch2, $op1, $op2\n\t"
15291             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15292             "b$cmp $labl" %}
15293   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15294   ins_encode %{
15295     Label* L = $labl$$label;
15296     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15297     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15298     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15299     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15300     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15301   %}
15302 
15303   ins_pipe(pipe_serial);
15304 %}
15305 
15306 // ============================================================================
15307 // Compare Instructions
15308 
15309 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15310 %{
15311   match(Set cr (CmpI op1 op2));
15312 
15313   effect(DEF cr, USE op1, USE op2);
15314 
15315   ins_cost(INSN_COST);
15316   format %{ "cmpw  $op1, $op2" %}
15317 
15318   ins_encode(aarch64_enc_cmpw(op1, op2));
15319 
15320   ins_pipe(icmp_reg_reg);
15321 %}
15322 
15323 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15324 %{
15325   match(Set cr (CmpI op1 zero));
15326 
15327   effect(DEF cr, USE op1);
15328 
15329   ins_cost(INSN_COST);
15330   format %{ "cmpw $op1, 0" %}
15331 
15332   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15333 
15334   ins_pipe(icmp_reg_imm);
15335 %}
15336 
15337 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15338 %{
15339   match(Set cr (CmpI op1 op2));
15340 
15341   effect(DEF cr, USE op1);
15342 
15343   ins_cost(INSN_COST);
15344   format %{ "cmpw  $op1, $op2" %}
15345 
15346   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15347 
15348   ins_pipe(icmp_reg_imm);
15349 %}
15350 
15351 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15352 %{
15353   match(Set cr (CmpI op1 op2));
15354 
15355   effect(DEF cr, USE op1);
15356 
15357   ins_cost(INSN_COST * 2);
15358   format %{ "cmpw  $op1, $op2" %}
15359 
15360   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15361 
15362   ins_pipe(icmp_reg_imm);
15363 %}
15364 
15365 // Unsigned compare Instructions; really, same as signed compare
15366 // except it should only be used to feed an If or a CMovI which takes a
15367 // cmpOpU.
15368 
15369 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15370 %{
15371   match(Set cr (CmpU op1 op2));
15372 
15373   effect(DEF cr, USE op1, USE op2);
15374 
15375   ins_cost(INSN_COST);
15376   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15377 
15378   ins_encode(aarch64_enc_cmpw(op1, op2));
15379 
15380   ins_pipe(icmp_reg_reg);
15381 %}
15382 
15383 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15384 %{
15385   match(Set cr (CmpU op1 zero));
15386 
15387   effect(DEF cr, USE op1);
15388 
15389   ins_cost(INSN_COST);
15390   format %{ "cmpw $op1, #0\t# unsigned" %}
15391 
15392   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15393 
15394   ins_pipe(icmp_reg_imm);
15395 %}
15396 
15397 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15398 %{
15399   match(Set cr (CmpU op1 op2));
15400 
15401   effect(DEF cr, USE op1);
15402 
15403   ins_cost(INSN_COST);
15404   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15405 
15406   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15407 
15408   ins_pipe(icmp_reg_imm);
15409 %}
15410 
15411 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15412 %{
15413   match(Set cr (CmpU op1 op2));
15414 
15415   effect(DEF cr, USE op1);
15416 
15417   ins_cost(INSN_COST * 2);
15418   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15419 
15420   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15421 
15422   ins_pipe(icmp_reg_imm);
15423 %}
15424 
15425 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15426 %{
15427   match(Set cr (CmpL op1 op2));
15428 
15429   effect(DEF cr, USE op1, USE op2);
15430 
15431   ins_cost(INSN_COST);
15432   format %{ "cmp  $op1, $op2" %}
15433 
15434   ins_encode(aarch64_enc_cmp(op1, op2));
15435 
15436   ins_pipe(icmp_reg_reg);
15437 %}
15438 
15439 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15440 %{
15441   match(Set cr (CmpL op1 zero));
15442 
15443   effect(DEF cr, USE op1);
15444 
15445   ins_cost(INSN_COST);
15446   format %{ "tst  $op1" %}
15447 
15448   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15449 
15450   ins_pipe(icmp_reg_imm);
15451 %}
15452 
15453 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15454 %{
15455   match(Set cr (CmpL op1 op2));
15456 
15457   effect(DEF cr, USE op1);
15458 
15459   ins_cost(INSN_COST);
15460   format %{ "cmp  $op1, $op2" %}
15461 
15462   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15463 
15464   ins_pipe(icmp_reg_imm);
15465 %}
15466 
15467 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15468 %{
15469   match(Set cr (CmpL op1 op2));
15470 
15471   effect(DEF cr, USE op1);
15472 
15473   ins_cost(INSN_COST * 2);
15474   format %{ "cmp  $op1, $op2" %}
15475 
15476   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15477 
15478   ins_pipe(icmp_reg_imm);
15479 %}
15480 
15481 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15482 %{
15483   match(Set cr (CmpUL op1 op2));
15484 
15485   effect(DEF cr, USE op1, USE op2);
15486 
15487   ins_cost(INSN_COST);
15488   format %{ "cmp  $op1, $op2" %}
15489 
15490   ins_encode(aarch64_enc_cmp(op1, op2));
15491 
15492   ins_pipe(icmp_reg_reg);
15493 %}
15494 
15495 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15496 %{
15497   match(Set cr (CmpUL op1 zero));
15498 
15499   effect(DEF cr, USE op1);
15500 
15501   ins_cost(INSN_COST);
15502   format %{ "tst  $op1" %}
15503 
15504   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15505 
15506   ins_pipe(icmp_reg_imm);
15507 %}
15508 
15509 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15510 %{
15511   match(Set cr (CmpUL op1 op2));
15512 
15513   effect(DEF cr, USE op1);
15514 
15515   ins_cost(INSN_COST);
15516   format %{ "cmp  $op1, $op2" %}
15517 
15518   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15519 
15520   ins_pipe(icmp_reg_imm);
15521 %}
15522 
15523 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15524 %{
15525   match(Set cr (CmpUL op1 op2));
15526 
15527   effect(DEF cr, USE op1);
15528 
15529   ins_cost(INSN_COST * 2);
15530   format %{ "cmp  $op1, $op2" %}
15531 
15532   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15533 
15534   ins_pipe(icmp_reg_imm);
15535 %}
15536 
15537 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15538 %{
15539   match(Set cr (CmpP op1 op2));
15540 
15541   effect(DEF cr, USE op1, USE op2);
15542 
15543   ins_cost(INSN_COST);
15544   format %{ "cmp  $op1, $op2\t // ptr" %}
15545 
15546   ins_encode(aarch64_enc_cmpp(op1, op2));
15547 
15548   ins_pipe(icmp_reg_reg);
15549 %}
15550 
15551 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15552 %{
15553   match(Set cr (CmpN op1 op2));
15554 
15555   effect(DEF cr, USE op1, USE op2);
15556 
15557   ins_cost(INSN_COST);
15558   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15559 
15560   ins_encode(aarch64_enc_cmpn(op1, op2));
15561 
15562   ins_pipe(icmp_reg_reg);
15563 %}
15564 
15565 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15566 %{
15567   match(Set cr (CmpP op1 zero));
15568 
15569   effect(DEF cr, USE op1, USE zero);
15570 
15571   ins_cost(INSN_COST);
15572   format %{ "cmp  $op1, 0\t // ptr" %}
15573 
15574   ins_encode(aarch64_enc_testp(op1));
15575 
15576   ins_pipe(icmp_reg_imm);
15577 %}
15578 
15579 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15580 %{
15581   match(Set cr (CmpN op1 zero));
15582 
15583   effect(DEF cr, USE op1, USE zero);
15584 
15585   ins_cost(INSN_COST);
15586   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15587 
15588   ins_encode(aarch64_enc_testn(op1));
15589 
15590   ins_pipe(icmp_reg_imm);
15591 %}
15592 
15593 // FP comparisons
15594 //
15595 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15596 // using normal cmpOp. See declaration of rFlagsReg for details.
15597 
15598 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15599 %{
15600   match(Set cr (CmpF src1 src2));
15601 
15602   ins_cost(3 * INSN_COST);
15603   format %{ "fcmps $src1, $src2" %}
15604 
15605   ins_encode %{
15606     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15607   %}
15608 
15609   ins_pipe(pipe_class_compare);
15610 %}
15611 
15612 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15613 %{
15614   match(Set cr (CmpF src1 src2));
15615 
15616   ins_cost(3 * INSN_COST);
15617   format %{ "fcmps $src1, 0.0" %}
15618 
15619   ins_encode %{
15620     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15621   %}
15622 
15623   ins_pipe(pipe_class_compare);
15624 %}
15625 // FROM HERE
15626 
15627 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15628 %{
15629   match(Set cr (CmpD src1 src2));
15630 
15631   ins_cost(3 * INSN_COST);
15632   format %{ "fcmpd $src1, $src2" %}
15633 
15634   ins_encode %{
15635     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15636   %}
15637 
15638   ins_pipe(pipe_class_compare);
15639 %}
15640 
15641 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15642 %{
15643   match(Set cr (CmpD src1 src2));
15644 
15645   ins_cost(3 * INSN_COST);
15646   format %{ "fcmpd $src1, 0.0" %}
15647 
15648   ins_encode %{
15649     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15650   %}
15651 
15652   ins_pipe(pipe_class_compare);
15653 %}
15654 
15655 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15656 %{
15657   match(Set dst (CmpF3 src1 src2));
15658   effect(KILL cr);
15659 
15660   ins_cost(5 * INSN_COST);
15661   format %{ "fcmps $src1, $src2\n\t"
15662             "csinvw($dst, zr, zr, eq\n\t"
15663             "csnegw($dst, $dst, $dst, lt)"
15664   %}
15665 
15666   ins_encode %{
15667     Label done;
15668     FloatRegister s1 = as_FloatRegister($src1$$reg);
15669     FloatRegister s2 = as_FloatRegister($src2$$reg);
15670     Register d = as_Register($dst$$reg);
15671     __ fcmps(s1, s2);
15672     // installs 0 if EQ else -1
15673     __ csinvw(d, zr, zr, Assembler::EQ);
15674     // keeps -1 if less or unordered else installs 1
15675     __ csnegw(d, d, d, Assembler::LT);
15676     __ bind(done);
15677   %}
15678 
15679   ins_pipe(pipe_class_default);
15680 
15681 %}
15682 
15683 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15684 %{
15685   match(Set dst (CmpD3 src1 src2));
15686   effect(KILL cr);
15687 
15688   ins_cost(5 * INSN_COST);
15689   format %{ "fcmpd $src1, $src2\n\t"
15690             "csinvw($dst, zr, zr, eq\n\t"
15691             "csnegw($dst, $dst, $dst, lt)"
15692   %}
15693 
15694   ins_encode %{
15695     Label done;
15696     FloatRegister s1 = as_FloatRegister($src1$$reg);
15697     FloatRegister s2 = as_FloatRegister($src2$$reg);
15698     Register d = as_Register($dst$$reg);
15699     __ fcmpd(s1, s2);
15700     // installs 0 if EQ else -1
15701     __ csinvw(d, zr, zr, Assembler::EQ);
15702     // keeps -1 if less or unordered else installs 1
15703     __ csnegw(d, d, d, Assembler::LT);
15704     __ bind(done);
15705   %}
15706   ins_pipe(pipe_class_default);
15707 
15708 %}
15709 
15710 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15711 %{
15712   match(Set dst (CmpF3 src1 zero));
15713   effect(KILL cr);
15714 
15715   ins_cost(5 * INSN_COST);
15716   format %{ "fcmps $src1, 0.0\n\t"
15717             "csinvw($dst, zr, zr, eq\n\t"
15718             "csnegw($dst, $dst, $dst, lt)"
15719   %}
15720 
15721   ins_encode %{
15722     Label done;
15723     FloatRegister s1 = as_FloatRegister($src1$$reg);
15724     Register d = as_Register($dst$$reg);
15725     __ fcmps(s1, 0.0);
15726     // installs 0 if EQ else -1
15727     __ csinvw(d, zr, zr, Assembler::EQ);
15728     // keeps -1 if less or unordered else installs 1
15729     __ csnegw(d, d, d, Assembler::LT);
15730     __ bind(done);
15731   %}
15732 
15733   ins_pipe(pipe_class_default);
15734 
15735 %}
15736 
15737 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15738 %{
15739   match(Set dst (CmpD3 src1 zero));
15740   effect(KILL cr);
15741 
15742   ins_cost(5 * INSN_COST);
15743   format %{ "fcmpd $src1, 0.0\n\t"
15744             "csinvw($dst, zr, zr, eq\n\t"
15745             "csnegw($dst, $dst, $dst, lt)"
15746   %}
15747 
15748   ins_encode %{
15749     Label done;
15750     FloatRegister s1 = as_FloatRegister($src1$$reg);
15751     Register d = as_Register($dst$$reg);
15752     __ fcmpd(s1, 0.0);
15753     // installs 0 if EQ else -1
15754     __ csinvw(d, zr, zr, Assembler::EQ);
15755     // keeps -1 if less or unordered else installs 1
15756     __ csnegw(d, d, d, Assembler::LT);
15757     __ bind(done);
15758   %}
15759   ins_pipe(pipe_class_default);
15760 
15761 %}
15762 
15763 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15764 %{
15765   match(Set dst (CmpLTMask p q));
15766   effect(KILL cr);
15767 
15768   ins_cost(3 * INSN_COST);
15769 
15770   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15771             "csetw $dst, lt\n\t"
15772             "subw $dst, zr, $dst"
15773   %}
15774 
15775   ins_encode %{
15776     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15777     __ csetw(as_Register($dst$$reg), Assembler::LT);
15778     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15779   %}
15780 
15781   ins_pipe(ialu_reg_reg);
15782 %}
15783 
15784 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15785 %{
15786   match(Set dst (CmpLTMask src zero));
15787   effect(KILL cr);
15788 
15789   ins_cost(INSN_COST);
15790 
15791   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15792 
15793   ins_encode %{
15794     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15795   %}
15796 
15797   ins_pipe(ialu_reg_shift);
15798 %}
15799 
15800 // ============================================================================
15801 // Max and Min
15802 
15803 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15804 %{
15805   effect( DEF dst, USE src1, USE src2, USE cr );
15806 
15807   ins_cost(INSN_COST * 2);
15808   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15809 
15810   ins_encode %{
15811     __ cselw(as_Register($dst$$reg),
15812              as_Register($src1$$reg),
15813              as_Register($src2$$reg),
15814              Assembler::LT);
15815   %}
15816 
15817   ins_pipe(icond_reg_reg);
15818 %}
15819 
15820 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15821 %{
15822   match(Set dst (MinI src1 src2));
15823   ins_cost(INSN_COST * 3);
15824 
15825   expand %{
15826     rFlagsReg cr;
15827     compI_reg_reg(cr, src1, src2);
15828     cmovI_reg_reg_lt(dst, src1, src2, cr);
15829   %}
15830 
15831 %}
15832 // FROM HERE
15833 
15834 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15835 %{
15836   effect( DEF dst, USE src1, USE src2, USE cr );
15837 
15838   ins_cost(INSN_COST * 2);
15839   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15840 
15841   ins_encode %{
15842     __ cselw(as_Register($dst$$reg),
15843              as_Register($src1$$reg),
15844              as_Register($src2$$reg),
15845              Assembler::GT);
15846   %}
15847 
15848   ins_pipe(icond_reg_reg);
15849 %}
15850 
15851 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15852 %{
15853   match(Set dst (MaxI src1 src2));
15854   ins_cost(INSN_COST * 3);
15855   expand %{
15856     rFlagsReg cr;
15857     compI_reg_reg(cr, src1, src2);
15858     cmovI_reg_reg_gt(dst, src1, src2, cr);
15859   %}
15860 %}
15861 
15862 // ============================================================================
15863 // Branch Instructions
15864 
15865 // Direct Branch.
15866 instruct branch(label lbl)
15867 %{
15868   match(Goto);
15869 
15870   effect(USE lbl);
15871 
15872   ins_cost(BRANCH_COST);
15873   format %{ "b  $lbl" %}
15874 
15875   ins_encode(aarch64_enc_b(lbl));
15876 
15877   ins_pipe(pipe_branch);
15878 %}
15879 
15880 // Conditional Near Branch
15881 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15882 %{
15883   // Same match rule as `branchConFar'.
15884   match(If cmp cr);
15885 
15886   effect(USE lbl);
15887 
15888   ins_cost(BRANCH_COST);
15889   // If set to 1 this indicates that the current instruction is a
15890   // short variant of a long branch. This avoids using this
15891   // instruction in first-pass matching. It will then only be used in
15892   // the `Shorten_branches' pass.
15893   // ins_short_branch(1);
15894   format %{ "b$cmp  $lbl" %}
15895 
15896   ins_encode(aarch64_enc_br_con(cmp, lbl));
15897 
15898   ins_pipe(pipe_branch_cond);
15899 %}
15900 
15901 // Conditional Near Branch Unsigned
15902 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15903 %{
15904   // Same match rule as `branchConFar'.
15905   match(If cmp cr);
15906 
15907   effect(USE lbl);
15908 
15909   ins_cost(BRANCH_COST);
15910   // If set to 1 this indicates that the current instruction is a
15911   // short variant of a long branch. This avoids using this
15912   // instruction in first-pass matching. It will then only be used in
15913   // the `Shorten_branches' pass.
15914   // ins_short_branch(1);
15915   format %{ "b$cmp  $lbl\t# unsigned" %}
15916 
15917   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15918 
15919   ins_pipe(pipe_branch_cond);
15920 %}
15921 
15922 // Make use of CBZ and CBNZ.  These instructions, as well as being
15923 // shorter than (cmp; branch), have the additional benefit of not
15924 // killing the flags.
15925 
15926 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15927   match(If cmp (CmpI op1 op2));
15928   effect(USE labl);
15929 
15930   ins_cost(BRANCH_COST);
15931   format %{ "cbw$cmp   $op1, $labl" %}
15932   ins_encode %{
15933     Label* L = $labl$$label;
15934     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15935     if (cond == Assembler::EQ)
15936       __ cbzw($op1$$Register, *L);
15937     else
15938       __ cbnzw($op1$$Register, *L);
15939   %}
15940   ins_pipe(pipe_cmp_branch);
15941 %}
15942 
15943 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15944   match(If cmp (CmpL op1 op2));
15945   effect(USE labl);
15946 
15947   ins_cost(BRANCH_COST);
15948   format %{ "cb$cmp   $op1, $labl" %}
15949   ins_encode %{
15950     Label* L = $labl$$label;
15951     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15952     if (cond == Assembler::EQ)
15953       __ cbz($op1$$Register, *L);
15954     else
15955       __ cbnz($op1$$Register, *L);
15956   %}
15957   ins_pipe(pipe_cmp_branch);
15958 %}
15959 
15960 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15961   match(If cmp (CmpP op1 op2));
15962   effect(USE labl);
15963 
15964   ins_cost(BRANCH_COST);
15965   format %{ "cb$cmp   $op1, $labl" %}
15966   ins_encode %{
15967     Label* L = $labl$$label;
15968     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15969     if (cond == Assembler::EQ)
15970       __ cbz($op1$$Register, *L);
15971     else
15972       __ cbnz($op1$$Register, *L);
15973   %}
15974   ins_pipe(pipe_cmp_branch);
15975 %}
15976 
15977 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15978   match(If cmp (CmpN op1 op2));
15979   effect(USE labl);
15980 
15981   ins_cost(BRANCH_COST);
15982   format %{ "cbw$cmp   $op1, $labl" %}
15983   ins_encode %{
15984     Label* L = $labl$$label;
15985     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15986     if (cond == Assembler::EQ)
15987       __ cbzw($op1$$Register, *L);
15988     else
15989       __ cbnzw($op1$$Register, *L);
15990   %}
15991   ins_pipe(pipe_cmp_branch);
15992 %}
15993 
15994 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15995   match(If cmp (CmpP (DecodeN oop) zero));
15996   effect(USE labl);
15997 
15998   ins_cost(BRANCH_COST);
15999   format %{ "cb$cmp   $oop, $labl" %}
16000   ins_encode %{
16001     Label* L = $labl$$label;
16002     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16003     if (cond == Assembler::EQ)
16004       __ cbzw($oop$$Register, *L);
16005     else
16006       __ cbnzw($oop$$Register, *L);
16007   %}
16008   ins_pipe(pipe_cmp_branch);
16009 %}
16010 
16011 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16012   match(If cmp (CmpU op1 op2));
16013   effect(USE labl);
16014 
16015   ins_cost(BRANCH_COST);
16016   format %{ "cbw$cmp   $op1, $labl" %}
16017   ins_encode %{
16018     Label* L = $labl$$label;
16019     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16020     if (cond == Assembler::EQ || cond == Assembler::LS)
16021       __ cbzw($op1$$Register, *L);
16022     else
16023       __ cbnzw($op1$$Register, *L);
16024   %}
16025   ins_pipe(pipe_cmp_branch);
16026 %}
16027 
16028 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16029   match(If cmp (CmpUL op1 op2));
16030   effect(USE labl);
16031 
16032   ins_cost(BRANCH_COST);
16033   format %{ "cb$cmp   $op1, $labl" %}
16034   ins_encode %{
16035     Label* L = $labl$$label;
16036     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16037     if (cond == Assembler::EQ || cond == Assembler::LS)
16038       __ cbz($op1$$Register, *L);
16039     else
16040       __ cbnz($op1$$Register, *L);
16041   %}
16042   ins_pipe(pipe_cmp_branch);
16043 %}
16044 
16045 // Test bit and Branch
16046 
16047 // Patterns for short (< 32KiB) variants
16048 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16049   match(If cmp (CmpL op1 op2));
16050   effect(USE labl);
16051 
16052   ins_cost(BRANCH_COST);
16053   format %{ "cb$cmp   $op1, $labl # long" %}
16054   ins_encode %{
16055     Label* L = $labl$$label;
16056     Assembler::Condition cond =
16057       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16058     __ tbr(cond, $op1$$Register, 63, *L);
16059   %}
16060   ins_pipe(pipe_cmp_branch);
16061   ins_short_branch(1);
16062 %}
16063 
16064 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16065   match(If cmp (CmpI op1 op2));
16066   effect(USE labl);
16067 
16068   ins_cost(BRANCH_COST);
16069   format %{ "cb$cmp   $op1, $labl # int" %}
16070   ins_encode %{
16071     Label* L = $labl$$label;
16072     Assembler::Condition cond =
16073       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16074     __ tbr(cond, $op1$$Register, 31, *L);
16075   %}
16076   ins_pipe(pipe_cmp_branch);
16077   ins_short_branch(1);
16078 %}
16079 
16080 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16081   match(If cmp (CmpL (AndL op1 op2) op3));
16082   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16083   effect(USE labl);
16084 
16085   ins_cost(BRANCH_COST);
16086   format %{ "tb$cmp   $op1, $op2, $labl" %}
16087   ins_encode %{
16088     Label* L = $labl$$label;
16089     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16090     int bit = exact_log2_long($op2$$constant);
16091     __ tbr(cond, $op1$$Register, bit, *L);
16092   %}
16093   ins_pipe(pipe_cmp_branch);
16094   ins_short_branch(1);
16095 %}
16096 
16097 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16098   match(If cmp (CmpI (AndI op1 op2) op3));
16099   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16100   effect(USE labl);
16101 
16102   ins_cost(BRANCH_COST);
16103   format %{ "tb$cmp   $op1, $op2, $labl" %}
16104   ins_encode %{
16105     Label* L = $labl$$label;
16106     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16107     int bit = exact_log2((juint)$op2$$constant);
16108     __ tbr(cond, $op1$$Register, bit, *L);
16109   %}
16110   ins_pipe(pipe_cmp_branch);
16111   ins_short_branch(1);
16112 %}
16113 
16114 // And far variants
16115 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16116   match(If cmp (CmpL op1 op2));
16117   effect(USE labl);
16118 
16119   ins_cost(BRANCH_COST);
16120   format %{ "cb$cmp   $op1, $labl # long" %}
16121   ins_encode %{
16122     Label* L = $labl$$label;
16123     Assembler::Condition cond =
16124       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16125     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16126   %}
16127   ins_pipe(pipe_cmp_branch);
16128 %}
16129 
16130 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16131   match(If cmp (CmpI op1 op2));
16132   effect(USE labl);
16133 
16134   ins_cost(BRANCH_COST);
16135   format %{ "cb$cmp   $op1, $labl # int" %}
16136   ins_encode %{
16137     Label* L = $labl$$label;
16138     Assembler::Condition cond =
16139       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16140     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16141   %}
16142   ins_pipe(pipe_cmp_branch);
16143 %}
16144 
16145 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16146   match(If cmp (CmpL (AndL op1 op2) op3));
16147   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16148   effect(USE labl);
16149 
16150   ins_cost(BRANCH_COST);
16151   format %{ "tb$cmp   $op1, $op2, $labl" %}
16152   ins_encode %{
16153     Label* L = $labl$$label;
16154     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16155     int bit = exact_log2_long($op2$$constant);
16156     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16157   %}
16158   ins_pipe(pipe_cmp_branch);
16159 %}
16160 
16161 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16162   match(If cmp (CmpI (AndI op1 op2) op3));
16163   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16164   effect(USE labl);
16165 
16166   ins_cost(BRANCH_COST);
16167   format %{ "tb$cmp   $op1, $op2, $labl" %}
16168   ins_encode %{
16169     Label* L = $labl$$label;
16170     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16171     int bit = exact_log2((juint)$op2$$constant);
16172     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16173   %}
16174   ins_pipe(pipe_cmp_branch);
16175 %}
16176 
16177 // Test bits
16178 
16179 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16180   match(Set cr (CmpL (AndL op1 op2) op3));
16181   predicate(Assembler::operand_valid_for_logical_immediate
16182             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16183 
16184   ins_cost(INSN_COST);
16185   format %{ "tst $op1, $op2 # long" %}
16186   ins_encode %{
16187     __ tst($op1$$Register, $op2$$constant);
16188   %}
16189   ins_pipe(ialu_reg_reg);
16190 %}
16191 
16192 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16193   match(Set cr (CmpI (AndI op1 op2) op3));
16194   predicate(Assembler::operand_valid_for_logical_immediate
16195             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16196 
16197   ins_cost(INSN_COST);
16198   format %{ "tst $op1, $op2 # int" %}
16199   ins_encode %{
16200     __ tstw($op1$$Register, $op2$$constant);
16201   %}
16202   ins_pipe(ialu_reg_reg);
16203 %}
16204 
16205 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16206   match(Set cr (CmpL (AndL op1 op2) op3));
16207 
16208   ins_cost(INSN_COST);
16209   format %{ "tst $op1, $op2 # long" %}
16210   ins_encode %{
16211     __ tst($op1$$Register, $op2$$Register);
16212   %}
16213   ins_pipe(ialu_reg_reg);
16214 %}
16215 
16216 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16217   match(Set cr (CmpI (AndI op1 op2) op3));
16218 
16219   ins_cost(INSN_COST);
16220   format %{ "tstw $op1, $op2 # int" %}
16221   ins_encode %{
16222     __ tstw($op1$$Register, $op2$$Register);
16223   %}
16224   ins_pipe(ialu_reg_reg);
16225 %}
16226 
16227 
16228 // Conditional Far Branch
16229 // Conditional Far Branch Unsigned
16230 // TODO: fixme
16231 
16232 // counted loop end branch near
16233 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16234 %{
16235   match(CountedLoopEnd cmp cr);
16236 
16237   effect(USE lbl);
16238 
16239   ins_cost(BRANCH_COST);
16240   // short variant.
16241   // ins_short_branch(1);
16242   format %{ "b$cmp $lbl \t// counted loop end" %}
16243 
16244   ins_encode(aarch64_enc_br_con(cmp, lbl));
16245 
16246   ins_pipe(pipe_branch);
16247 %}
16248 
16249 // counted loop end branch near Unsigned
16250 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16251 %{
16252   match(CountedLoopEnd cmp cr);
16253 
16254   effect(USE lbl);
16255 
16256   ins_cost(BRANCH_COST);
16257   // short variant.
16258   // ins_short_branch(1);
16259   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16260 
16261   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16262 
16263   ins_pipe(pipe_branch);
16264 %}
16265 
16266 // counted loop end branch far
16267 // counted loop end branch far unsigned
16268 // TODO: fixme
16269 
16270 // ============================================================================
16271 // inlined locking and unlocking
16272 
16273 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16274 %{
16275   match(Set cr (FastLock object box));
16276   effect(TEMP tmp, TEMP tmp2);
16277 
16278   // TODO
16279   // identify correct cost
16280   ins_cost(5 * INSN_COST);
16281   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16282 
16283   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16284 
16285   ins_pipe(pipe_serial);
16286 %}
16287 
16288 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16289 %{
16290   match(Set cr (FastUnlock object box));
16291   effect(TEMP tmp, TEMP tmp2);
16292 
16293   ins_cost(5 * INSN_COST);
16294   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16295 
16296   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16297 
16298   ins_pipe(pipe_serial);
16299 %}
16300 
16301 
16302 // ============================================================================
16303 // Safepoint Instructions
16304 
16305 // TODO
16306 // provide a near and far version of this code
16307 
16308 instruct safePoint(rFlagsReg cr, iRegP poll)
16309 %{
16310   match(SafePoint poll);
16311   effect(KILL cr);
16312 
16313   format %{
16314     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16315   %}
16316   ins_encode %{
16317     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16318   %}
16319   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16320 %}
16321 
16322 
16323 // ============================================================================
16324 // Procedure Call/Return Instructions
16325 
16326 // Call Java Static Instruction
16327 
16328 instruct CallStaticJavaDirect(method meth)
16329 %{
16330   match(CallStaticJava);
16331 
16332   effect(USE meth);
16333 
16334   ins_cost(CALL_COST);
16335 
16336   format %{ "call,static $meth \t// ==> " %}
16337 
16338   ins_encode(aarch64_enc_java_static_call(meth),
16339              aarch64_enc_call_epilog);
16340 
16341   ins_pipe(pipe_class_call);
16342 %}
16343 
16344 // TO HERE
16345 
16346 // Call Java Dynamic Instruction
16347 instruct CallDynamicJavaDirect(method meth)
16348 %{
16349   match(CallDynamicJava);
16350 
16351   effect(USE meth);
16352 
16353   ins_cost(CALL_COST);
16354 
16355   format %{ "CALL,dynamic $meth \t// ==> " %}
16356 
16357   ins_encode(aarch64_enc_java_dynamic_call(meth),
16358              aarch64_enc_call_epilog);
16359 
16360   ins_pipe(pipe_class_call);
16361 %}
16362 
16363 // Call Runtime Instruction
16364 
16365 instruct CallRuntimeDirect(method meth)
16366 %{
16367   match(CallRuntime);
16368 
16369   effect(USE meth);
16370 
16371   ins_cost(CALL_COST);
16372 
16373   format %{ "CALL, runtime $meth" %}
16374 
16375   ins_encode( aarch64_enc_java_to_runtime(meth) );
16376 
16377   ins_pipe(pipe_class_call);
16378 %}
16379 
16380 // Call Runtime Instruction
16381 
16382 instruct CallLeafDirect(method meth)
16383 %{
16384   match(CallLeaf);
16385 
16386   effect(USE meth);
16387 
16388   ins_cost(CALL_COST);
16389 
16390   format %{ "CALL, runtime leaf $meth" %}
16391 
16392   ins_encode( aarch64_enc_java_to_runtime(meth) );
16393 
16394   ins_pipe(pipe_class_call);
16395 %}
16396 
16397 // Call Runtime Instruction
16398 
16399 // entry point is null, target holds the address to call
16400 instruct CallLeafNoFPIndirect(iRegP target)
16401 %{
16402   predicate(n->as_Call()->entry_point() == NULL);
16403 
16404   match(CallLeafNoFP target);
16405 
16406   ins_cost(CALL_COST);
16407 
16408   format %{ "CALL, runtime leaf nofp indirect $target" %}
16409 
16410   ins_encode %{
16411     __ blr($target$$Register);
16412   %}
16413 
16414   ins_pipe(pipe_class_call);
16415 %}
16416 
16417 instruct CallLeafNoFPDirect(method meth)
16418 %{
16419   predicate(n->as_Call()->entry_point() != NULL);
16420 
16421   match(CallLeafNoFP);
16422 
16423   effect(USE meth);
16424 
16425   ins_cost(CALL_COST);
16426 
16427   format %{ "CALL, runtime leaf nofp $meth" %}
16428 
16429   ins_encode( aarch64_enc_java_to_runtime(meth) );
16430 
16431   ins_pipe(pipe_class_call);
16432 %}
16433 
16434 instruct CallNativeDirect(method meth)
16435 %{
16436   match(CallNative);
16437 
16438   effect(USE meth);
16439 
16440   ins_cost(CALL_COST);
16441 
16442   format %{ "CALL, native $meth" %}
16443 
16444   ins_encode( aarch64_enc_java_to_runtime(meth) );
16445 
16446   ins_pipe(pipe_class_call);
16447 %}
16448 
16449 // Tail Call; Jump from runtime stub to Java code.
16450 // Also known as an 'interprocedural jump'.
16451 // Target of jump will eventually return to caller.
16452 // TailJump below removes the return address.
16453 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16454 %{
16455   match(TailCall jump_target method_ptr);
16456 
16457   ins_cost(CALL_COST);
16458 
16459   format %{ "br $jump_target\t# $method_ptr holds method" %}
16460 
16461   ins_encode(aarch64_enc_tail_call(jump_target));
16462 
16463   ins_pipe(pipe_class_call);
16464 %}
16465 
16466 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16467 %{
16468   match(TailJump jump_target ex_oop);
16469 
16470   ins_cost(CALL_COST);
16471 
16472   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16473 
16474   ins_encode(aarch64_enc_tail_jmp(jump_target));
16475 
16476   ins_pipe(pipe_class_call);
16477 %}
16478 
16479 // Create exception oop: created by stack-crawling runtime code.
16480 // Created exception is now available to this handler, and is setup
16481 // just prior to jumping to this handler. No code emitted.
16482 // TODO check
16483 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16484 instruct CreateException(iRegP_R0 ex_oop)
16485 %{
16486   match(Set ex_oop (CreateEx));
16487 
16488   format %{ " -- \t// exception oop; no code emitted" %}
16489 
16490   size(0);
16491 
16492   ins_encode( /*empty*/ );
16493 
16494   ins_pipe(pipe_class_empty);
16495 %}
16496 
16497 // Rethrow exception: The exception oop will come in the first
16498 // argument position. Then JUMP (not call) to the rethrow stub code.
16499 instruct RethrowException() %{
16500   match(Rethrow);
16501   ins_cost(CALL_COST);
16502 
16503   format %{ "b rethrow_stub" %}
16504 
16505   ins_encode( aarch64_enc_rethrow() );
16506 
16507   ins_pipe(pipe_class_call);
16508 %}
16509 
16510 
16511 // Return Instruction
16512 // epilog node loads ret address into lr as part of frame pop
16513 instruct Ret()
16514 %{
16515   match(Return);
16516 
16517   format %{ "ret\t// return register" %}
16518 
16519   ins_encode( aarch64_enc_ret() );
16520 
16521   ins_pipe(pipe_branch);
16522 %}
16523 
16524 // Die now.
16525 instruct ShouldNotReachHere() %{
16526   match(Halt);
16527 
16528   ins_cost(CALL_COST);
16529   format %{ "ShouldNotReachHere" %}
16530 
16531   ins_encode %{
16532     if (is_reachable()) {
16533       __ stop(_halt_reason);
16534     }
16535   %}
16536 
16537   ins_pipe(pipe_class_default);
16538 %}
16539 
16540 // ============================================================================
16541 // Partial Subtype Check
16542 //
16543 // superklass array for an instance of the superklass.  Set a hidden
16544 // internal cache on a hit (cache is checked with exposed code in
16545 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16546 // encoding ALSO sets flags.
16547 
16548 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16549 %{
16550   match(Set result (PartialSubtypeCheck sub super));
16551   effect(KILL cr, KILL temp);
16552 
16553   ins_cost(1100);  // slightly larger than the next version
16554   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16555 
16556   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16557 
16558   opcode(0x1); // Force zero of result reg on hit
16559 
16560   ins_pipe(pipe_class_memory);
16561 %}
16562 
16563 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16564 %{
16565   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16566   effect(KILL temp, KILL result);
16567 
16568   ins_cost(1100);  // slightly larger than the next version
16569   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16570 
16571   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16572 
16573   opcode(0x0); // Don't zero result reg on hit
16574 
16575   ins_pipe(pipe_class_memory);
16576 %}
16577 
16578 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16579                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16580 %{
16581   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16582   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16583   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16584 
16585   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16586   ins_encode %{
16587     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16588     __ string_compare($str1$$Register, $str2$$Register,
16589                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16590                       $tmp1$$Register, $tmp2$$Register,
16591                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16592   %}
16593   ins_pipe(pipe_class_memory);
16594 %}
16595 
16596 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16597                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16598 %{
16599   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16600   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16601   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16602 
16603   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16604   ins_encode %{
16605     __ string_compare($str1$$Register, $str2$$Register,
16606                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16607                       $tmp1$$Register, $tmp2$$Register,
16608                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16609   %}
16610   ins_pipe(pipe_class_memory);
16611 %}
16612 
16613 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16614                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16615                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16616 %{
16617   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16618   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16619   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16620          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16621 
16622   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16623   ins_encode %{
16624     __ string_compare($str1$$Register, $str2$$Register,
16625                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16626                       $tmp1$$Register, $tmp2$$Register,
16627                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16628                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16629   %}
16630   ins_pipe(pipe_class_memory);
16631 %}
16632 
16633 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16634                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16635                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16636 %{
16637   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16638   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16639   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16640          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16641 
16642   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16643   ins_encode %{
16644     __ string_compare($str1$$Register, $str2$$Register,
16645                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16646                       $tmp1$$Register, $tmp2$$Register,
16647                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16648                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16649   %}
16650   ins_pipe(pipe_class_memory);
16651 %}
16652 
16653 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16654        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16655        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16656 %{
16657   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16658   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16659   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16660          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16661   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16662 
16663   ins_encode %{
16664     __ string_indexof($str1$$Register, $str2$$Register,
16665                       $cnt1$$Register, $cnt2$$Register,
16666                       $tmp1$$Register, $tmp2$$Register,
16667                       $tmp3$$Register, $tmp4$$Register,
16668                       $tmp5$$Register, $tmp6$$Register,
16669                       -1, $result$$Register, StrIntrinsicNode::UU);
16670   %}
16671   ins_pipe(pipe_class_memory);
16672 %}
16673 
16674 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16675        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16676        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16677 %{
16678   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16679   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16680   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16681          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16682   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16683 
16684   ins_encode %{
16685     __ string_indexof($str1$$Register, $str2$$Register,
16686                       $cnt1$$Register, $cnt2$$Register,
16687                       $tmp1$$Register, $tmp2$$Register,
16688                       $tmp3$$Register, $tmp4$$Register,
16689                       $tmp5$$Register, $tmp6$$Register,
16690                       -1, $result$$Register, StrIntrinsicNode::LL);
16691   %}
16692   ins_pipe(pipe_class_memory);
16693 %}
16694 
16695 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16696        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16697        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16698 %{
16699   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16700   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16701   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16702          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16703   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16704 
16705   ins_encode %{
16706     __ string_indexof($str1$$Register, $str2$$Register,
16707                       $cnt1$$Register, $cnt2$$Register,
16708                       $tmp1$$Register, $tmp2$$Register,
16709                       $tmp3$$Register, $tmp4$$Register,
16710                       $tmp5$$Register, $tmp6$$Register,
16711                       -1, $result$$Register, StrIntrinsicNode::UL);
16712   %}
16713   ins_pipe(pipe_class_memory);
16714 %}
16715 
16716 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16717                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16718                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16719 %{
16720   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16721   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16722   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16723          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16724   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16725 
16726   ins_encode %{
16727     int icnt2 = (int)$int_cnt2$$constant;
16728     __ string_indexof($str1$$Register, $str2$$Register,
16729                       $cnt1$$Register, zr,
16730                       $tmp1$$Register, $tmp2$$Register,
16731                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16732                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16733   %}
16734   ins_pipe(pipe_class_memory);
16735 %}
16736 
16737 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16738                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16739                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16740 %{
16741   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16742   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16743   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16744          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16745   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16746 
16747   ins_encode %{
16748     int icnt2 = (int)$int_cnt2$$constant;
16749     __ string_indexof($str1$$Register, $str2$$Register,
16750                       $cnt1$$Register, zr,
16751                       $tmp1$$Register, $tmp2$$Register,
16752                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16753                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16754   %}
16755   ins_pipe(pipe_class_memory);
16756 %}
16757 
16758 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16759                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16760                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16761 %{
16762   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16763   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16764   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16765          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16766   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16767 
16768   ins_encode %{
16769     int icnt2 = (int)$int_cnt2$$constant;
16770     __ string_indexof($str1$$Register, $str2$$Register,
16771                       $cnt1$$Register, zr,
16772                       $tmp1$$Register, $tmp2$$Register,
16773                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16774                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16775   %}
16776   ins_pipe(pipe_class_memory);
16777 %}
16778 
16779 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16780                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16781                              iRegINoSp tmp3, rFlagsReg cr)
16782 %{
16783   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16784   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16785   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16786          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16787 
16788   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16789 
16790   ins_encode %{
16791     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16792                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16793                            $tmp3$$Register);
16794   %}
16795   ins_pipe(pipe_class_memory);
16796 %}
16797 
16798 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16799                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16800                               iRegINoSp tmp3, rFlagsReg cr)
16801 %{
16802   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16803   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16804   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16805          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16806 
16807   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16808 
16809   ins_encode %{
16810     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16811                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16812                             $tmp3$$Register);
16813   %}
16814   ins_pipe(pipe_class_memory);
16815 %}
16816 
16817 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16818                         iRegI_R0 result, rFlagsReg cr)
16819 %{
16820   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16821   match(Set result (StrEquals (Binary str1 str2) cnt));
16822   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16823 
16824   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16825   ins_encode %{
16826     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16827     __ string_equals($str1$$Register, $str2$$Register,
16828                      $result$$Register, $cnt$$Register, 1);
16829   %}
16830   ins_pipe(pipe_class_memory);
16831 %}
16832 
16833 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16834                         iRegI_R0 result, rFlagsReg cr)
16835 %{
16836   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16837   match(Set result (StrEquals (Binary str1 str2) cnt));
16838   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16839 
16840   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16841   ins_encode %{
16842     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16843     __ string_equals($str1$$Register, $str2$$Register,
16844                      $result$$Register, $cnt$$Register, 2);
16845   %}
16846   ins_pipe(pipe_class_memory);
16847 %}
16848 
16849 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16850                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16851                        iRegP_R10 tmp, rFlagsReg cr)
16852 %{
16853   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16854   match(Set result (AryEq ary1 ary2));
16855   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16856 
16857   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16858   ins_encode %{
16859     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16860                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16861                                    $result$$Register, $tmp$$Register, 1);
16862     if (tpc == NULL) {
16863       ciEnv::current()->record_failure("CodeCache is full");
16864       return;
16865     }
16866   %}
16867   ins_pipe(pipe_class_memory);
16868 %}
16869 
16870 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16871                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16872                        iRegP_R10 tmp, rFlagsReg cr)
16873 %{
16874   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16875   match(Set result (AryEq ary1 ary2));
16876   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16877 
16878   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16879   ins_encode %{
16880     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16881                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16882                                    $result$$Register, $tmp$$Register, 2);
16883     if (tpc == NULL) {
16884       ciEnv::current()->record_failure("CodeCache is full");
16885       return;
16886     }
16887   %}
16888   ins_pipe(pipe_class_memory);
16889 %}
16890 
16891 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16892 %{
16893   match(Set result (HasNegatives ary1 len));
16894   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16895   format %{ "has negatives byte[] $ary1,$len -> $result" %}
16896   ins_encode %{
16897     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16898     if (tpc == NULL) {
16899       ciEnv::current()->record_failure("CodeCache is full");
16900       return;
16901     }
16902   %}
16903   ins_pipe( pipe_slow );
16904 %}
16905 
16906 // fast char[] to byte[] compression
16907 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16908                          vRegD_V0 tmp1, vRegD_V1 tmp2,
16909                          vRegD_V2 tmp3, vRegD_V3 tmp4,
16910                          iRegI_R0 result, rFlagsReg cr)
16911 %{
16912   match(Set result (StrCompressedCopy src (Binary dst len)));
16913   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16914 
16915   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
16916   ins_encode %{
16917     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16918                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16919                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16920                            $result$$Register);
16921   %}
16922   ins_pipe( pipe_slow );
16923 %}
16924 
16925 // fast byte[] to char[] inflation
16926 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16927                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16928 %{
16929   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16930   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16931 
16932   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
16933   ins_encode %{
16934     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16935                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16936                                         $tmp3$$FloatRegister, $tmp4$$Register);
16937     if (tpc == NULL) {
16938       ciEnv::current()->record_failure("CodeCache is full");
16939       return;
16940     }
16941   %}
16942   ins_pipe(pipe_class_memory);
16943 %}
16944 
16945 // encode char[] to byte[] in ISO_8859_1
16946 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16947                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16948                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16949                           iRegI_R0 result, rFlagsReg cr)
16950 %{
16951   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16952   match(Set result (EncodeISOArray src (Binary dst len)));
16953   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16954          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16955 
16956   format %{ "Encode array $src,$dst,$len -> $result" %}
16957   ins_encode %{
16958     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16959          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
16960          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
16961   %}
16962   ins_pipe( pipe_class_memory );
16963 %}
16964 
16965 // ============================================================================
16966 // This name is KNOWN by the ADLC and cannot be changed.
16967 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16968 // for this guy.
16969 instruct tlsLoadP(thread_RegP dst)
16970 %{
16971   match(Set dst (ThreadLocal));
16972 
16973   ins_cost(0);
16974 
16975   format %{ " -- \t// $dst=Thread::current(), empty" %}
16976 
16977   size(0);
16978 
16979   ins_encode( /*empty*/ );
16980 
16981   ins_pipe(pipe_class_empty);
16982 %}
16983 
16984 //----------PEEPHOLE RULES-----------------------------------------------------
16985 // These must follow all instruction definitions as they use the names
16986 // defined in the instructions definitions.
16987 //
16988 // peepmatch ( root_instr_name [preceding_instruction]* );
16989 //
16990 // peepconstraint %{
16991 // (instruction_number.operand_name relational_op instruction_number.operand_name
16992 //  [, ...] );
16993 // // instruction numbers are zero-based using left to right order in peepmatch
16994 //
16995 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16996 // // provide an instruction_number.operand_name for each operand that appears
16997 // // in the replacement instruction's match rule
16998 //
16999 // ---------VM FLAGS---------------------------------------------------------
17000 //
17001 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17002 //
17003 // Each peephole rule is given an identifying number starting with zero and
17004 // increasing by one in the order seen by the parser.  An individual peephole
17005 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17006 // on the command-line.
17007 //
17008 // ---------CURRENT LIMITATIONS----------------------------------------------
17009 //
17010 // Only match adjacent instructions in same basic block
17011 // Only equality constraints
17012 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17013 // Only one replacement instruction
17014 //
17015 // ---------EXAMPLE----------------------------------------------------------
17016 //
17017 // // pertinent parts of existing instructions in architecture description
17018 // instruct movI(iRegINoSp dst, iRegI src)
17019 // %{
17020 //   match(Set dst (CopyI src));
17021 // %}
17022 //
17023 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17024 // %{
17025 //   match(Set dst (AddI dst src));
17026 //   effect(KILL cr);
17027 // %}
17028 //
17029 // // Change (inc mov) to lea
17030 // peephole %{
17031 //   // increment preceeded by register-register move
17032 //   peepmatch ( incI_iReg movI );
17033 //   // require that the destination register of the increment
17034 //   // match the destination register of the move
17035 //   peepconstraint ( 0.dst == 1.dst );
17036 //   // construct a replacement instruction that sets
17037 //   // the destination to ( move's source register + one )
17038 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17039 // %}
17040 //
17041 
17042 // Implementation no longer uses movX instructions since
17043 // machine-independent system no longer uses CopyX nodes.
17044 //
17045 // peephole
17046 // %{
17047 //   peepmatch (incI_iReg movI);
17048 //   peepconstraint (0.dst == 1.dst);
17049 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17050 // %}
17051 
17052 // peephole
17053 // %{
17054 //   peepmatch (decI_iReg movI);
17055 //   peepconstraint (0.dst == 1.dst);
17056 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17057 // %}
17058 
17059 // peephole
17060 // %{
17061 //   peepmatch (addI_iReg_imm movI);
17062 //   peepconstraint (0.dst == 1.dst);
17063 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17064 // %}
17065 
17066 // peephole
17067 // %{
17068 //   peepmatch (incL_iReg movL);
17069 //   peepconstraint (0.dst == 1.dst);
17070 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17071 // %}
17072 
17073 // peephole
17074 // %{
17075 //   peepmatch (decL_iReg movL);
17076 //   peepconstraint (0.dst == 1.dst);
17077 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17078 // %}
17079 
17080 // peephole
17081 // %{
17082 //   peepmatch (addL_iReg_imm movL);
17083 //   peepconstraint (0.dst == 1.dst);
17084 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17085 // %}
17086 
17087 // peephole
17088 // %{
17089 //   peepmatch (addP_iReg_imm movP);
17090 //   peepconstraint (0.dst == 1.dst);
17091 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17092 // %}
17093 
17094 // // Change load of spilled value to only a spill
17095 // instruct storeI(memory mem, iRegI src)
17096 // %{
17097 //   match(Set mem (StoreI mem src));
17098 // %}
17099 //
17100 // instruct loadI(iRegINoSp dst, memory mem)
17101 // %{
17102 //   match(Set dst (LoadI mem));
17103 // %}
17104 //
17105 
17106 //----------SMARTSPILL RULES---------------------------------------------------
17107 // These must follow all instruction definitions as they use the names
17108 // defined in the instructions definitions.
17109 
17110 // Local Variables:
17111 // mode: c++
17112 // End: