1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // archtecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182   reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
  183   reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
  184   reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
  185   reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
  186 
  187   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  188   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  189   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  190   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  191   reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
  192   reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
  193   reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
  194   reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
  195 
  196   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  197   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  198   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  199   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  200   reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
  201   reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
  202   reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
  203   reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
  204 
  205   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  206   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  207   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  208   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  209   reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
  210   reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
  211   reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
  212   reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
  213 
  214   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  215   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  216   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  217   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  218   reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
  219   reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
  220   reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
  221   reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
  222 
  223   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  224   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  225   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  226   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  227   reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
  228   reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
  229   reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
  230   reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
  231 
  232   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  233   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  234   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  235   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  236   reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
  237   reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
  238   reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
  239   reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
  240 
  241   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  242   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  243   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  244   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  245   reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
  246   reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
  247   reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
  248   reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
  249 
  250   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  251   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  252   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  253   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  254   reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
  255   reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
  256   reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
  257   reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
  258 
  259   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  260   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  261   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  262   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  263   reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
  264   reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
  265   reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
  266   reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
  267 
  268   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  269   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  270   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  271   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  272   reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
  273   reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
  274   reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
  275   reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
  276 
  277   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  278   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  279   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  280   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  281   reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
  282   reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
  283   reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
  284   reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
  285 
  286   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  287   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  288   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  289   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  290   reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
  291   reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
  292   reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
  293   reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
  294 
  295   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  296   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  297   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  298   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  299   reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
  300   reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
  301   reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
  302   reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
  303 
  304   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  305   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  306   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  307   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  308   reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
  309   reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
  310   reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
  311   reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
  312 
  313   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  314   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  315   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  316   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  317   reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
  318   reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
  319   reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
  320   reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
  321 
  322   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  323   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  324   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  325   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  326   reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
  327   reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
  328   reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
  329   reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
  330 
  331   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  332   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  333   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  334   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  335   reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
  336   reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
  337   reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
  338   reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
  339 
  340   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  341   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  342   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  343   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  344   reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
  345   reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
  346   reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
  347   reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
  348 
  349   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  350   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  351   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  352   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  353   reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
  354   reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
  355   reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
  356   reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
  357 
  358   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  359   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  360   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  361   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  362   reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
  363   reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
  364   reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
  365   reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
  366 
  367   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  368   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  369   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  370   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  371   reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
  372   reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
  373   reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
  374   reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
  375 
  376   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  377   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  378   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  379   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  380   reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
  381   reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
  382   reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
  383   reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
  384 
  385   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  386   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  387   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  388   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  389   reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
  390   reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
  391   reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
  392   reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
  393 
  394   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  395   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  396   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  397   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  398   reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
  399   reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
  400   reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
  401   reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
  402 
  403   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  404   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  405   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  406   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  407   reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
  408   reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
  409   reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
  410   reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
  411 
  412   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  413   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  414   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  415   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  416   reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
  417   reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
  418   reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
  419   reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
  420 
  421   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  422   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  423   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  424   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  425   reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
  426   reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
  427   reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
  428   reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
  429 
  430   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  431   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  432   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  433   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  434   reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
  435   reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
  436   reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
  437   reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
  438 
  439   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  440   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  441   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  442   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  443   reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
  444   reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
  445   reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
  446   reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
  447 
  448   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  449   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  450   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  451   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  452   reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
  453   reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
  454   reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
  455   reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
  456 
  457   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  458   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  459   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  460   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  461   reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
  462   reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
  463   reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
  464   reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
  465 
  466 
  467 // ----------------------------
  468 // SVE Predicate Registers
  469 // ----------------------------
  470   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  471   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  472   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  473   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  474   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  475   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  476   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  477   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  478   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  479   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  480   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  481   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  482   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  483   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  484   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  485   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  486 
  487 // ----------------------------
  488 // Special Registers
  489 // ----------------------------
  490 
  491 // the AArch64 CSPR status flag register is not directly acessible as
  492 // instruction operand. the FPSR status flag register is a system
  493 // register which can be written/read using MSR/MRS but again does not
  494 // appear as an operand (a code identifying the FSPR occurs as an
  495 // immediate value in the instruction).
  496 
  497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  498 
  499 // Specify priority of register selection within phases of register
  500 // allocation.  Highest priority is first.  A useful heuristic is to
  501 // give registers a low priority when they are required by machine
  502 // instructions, like EAX and EDX on I486, and choose no-save registers
  503 // before save-on-call, & save-on-call before save-on-entry.  Registers
  504 // which participate in fixed calling sequences should come last.
  505 // Registers which are used as pairs must fall on an even boundary.
  506 
  507 alloc_class chunk0(
  508     // volatiles
  509     R10, R10_H,
  510     R11, R11_H,
  511     R12, R12_H,
  512     R13, R13_H,
  513     R14, R14_H,
  514     R15, R15_H,
  515     R16, R16_H,
  516     R17, R17_H,
  517     R18, R18_H,
  518 
  519     // arg registers
  520     R0, R0_H,
  521     R1, R1_H,
  522     R2, R2_H,
  523     R3, R3_H,
  524     R4, R4_H,
  525     R5, R5_H,
  526     R6, R6_H,
  527     R7, R7_H,
  528 
  529     // non-volatiles
  530     R19, R19_H,
  531     R20, R20_H,
  532     R21, R21_H,
  533     R22, R22_H,
  534     R23, R23_H,
  535     R24, R24_H,
  536     R25, R25_H,
  537     R26, R26_H,
  538 
  539     // non-allocatable registers
  540 
  541     R27, R27_H, // heapbase
  542     R28, R28_H, // thread
  543     R29, R29_H, // fp
  544     R30, R30_H, // lr
  545     R31, R31_H, // sp
  546     R8, R8_H,   // rscratch1
  547     R9, R9_H,   // rscratch2
  548 );
  549 
  550 alloc_class chunk1(
  551 
  552     // no save
  553     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  554     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  555     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  556     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  557     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  558     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  559     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  560     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  561     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  562     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  563     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  564     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  565     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  566     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  567     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  568     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  569 
  570     // arg registers
  571     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  572     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  573     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  574     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  575     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  576     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  577     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  578     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  579 
  580     // non-volatiles
  581     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  582     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  583     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  584     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  585     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  586     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  587     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  588     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  589 );
  590 
  591 alloc_class chunk2 (
  592     P0,
  593     P1,
  594     P2,
  595     P3,
  596     P4,
  597     P5,
  598     P6,
  599     P7,
  600 
  601     P8,
  602     P9,
  603     P10,
  604     P11,
  605     P12,
  606     P13,
  607     P14,
  608     P15,
  609 );
  610 
  611 alloc_class chunk3(RFLAGS);
  612 
  613 //----------Architecture Description Register Classes--------------------------
  614 // Several register classes are automatically defined based upon information in
  615 // this architecture description.
  616 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  618 //
  619 
  620 // Class for all 32 bit general purpose registers
  621 reg_class all_reg32(
  622     R0,
  623     R1,
  624     R2,
  625     R3,
  626     R4,
  627     R5,
  628     R6,
  629     R7,
  630     R10,
  631     R11,
  632     R12,
  633     R13,
  634     R14,
  635     R15,
  636     R16,
  637     R17,
  638     R18,
  639     R19,
  640     R20,
  641     R21,
  642     R22,
  643     R23,
  644     R24,
  645     R25,
  646     R26,
  647     R27,
  648     R28,
  649     R29,
  650     R30,
  651     R31
  652 );
  653 
  654 
  655 // Class for all 32 bit integer registers (excluding SP which
  656 // will never be used as an integer register)
  657 reg_class any_reg32 %{
  658   return _ANY_REG32_mask;
  659 %}
  660 
  661 // Singleton class for R0 int register
  662 reg_class int_r0_reg(R0);
  663 
  664 // Singleton class for R2 int register
  665 reg_class int_r2_reg(R2);
  666 
  667 // Singleton class for R3 int register
  668 reg_class int_r3_reg(R3);
  669 
  670 // Singleton class for R4 int register
  671 reg_class int_r4_reg(R4);
  672 
  673 // Singleton class for R31 int register
  674 reg_class int_r31_reg(R31);
  675 
  676 // Class for all 64 bit general purpose registers
  677 reg_class all_reg(
  678     R0, R0_H,
  679     R1, R1_H,
  680     R2, R2_H,
  681     R3, R3_H,
  682     R4, R4_H,
  683     R5, R5_H,
  684     R6, R6_H,
  685     R7, R7_H,
  686     R10, R10_H,
  687     R11, R11_H,
  688     R12, R12_H,
  689     R13, R13_H,
  690     R14, R14_H,
  691     R15, R15_H,
  692     R16, R16_H,
  693     R17, R17_H,
  694     R18, R18_H,
  695     R19, R19_H,
  696     R20, R20_H,
  697     R21, R21_H,
  698     R22, R22_H,
  699     R23, R23_H,
  700     R24, R24_H,
  701     R25, R25_H,
  702     R26, R26_H,
  703     R27, R27_H,
  704     R28, R28_H,
  705     R29, R29_H,
  706     R30, R30_H,
  707     R31, R31_H
  708 );
  709 
  710 // Class for all long integer registers (including SP)
  711 reg_class any_reg %{
  712   return _ANY_REG_mask;
  713 %}
  714 
  715 // Class for non-allocatable 32 bit registers
  716 reg_class non_allocatable_reg32(
  717 #ifdef R18_RESERVED
  718     // See comment in register_aarch64.hpp
  719     R18,                        // tls on Windows
  720 #endif
  721     R28,                        // thread
  722     R30,                        // lr
  723     R31                         // sp
  724 );
  725 
  726 // Class for non-allocatable 64 bit registers
  727 reg_class non_allocatable_reg(
  728 #ifdef R18_RESERVED
  729     // See comment in register_aarch64.hpp
  730     R18, R18_H,                 // tls on Windows, platform register on macOS
  731 #endif
  732     R28, R28_H,                 // thread
  733     R30, R30_H,                 // lr
  734     R31, R31_H                  // sp
  735 );
  736 
  737 // Class for all non-special integer registers
  738 reg_class no_special_reg32 %{
  739   return _NO_SPECIAL_REG32_mask;
  740 %}
  741 
  742 // Class for all non-special long integer registers
  743 reg_class no_special_reg %{
  744   return _NO_SPECIAL_REG_mask;
  745 %}
  746 
  747 // Class for 64 bit register r0
  748 reg_class r0_reg(
  749     R0, R0_H
  750 );
  751 
  752 // Class for 64 bit register r1
  753 reg_class r1_reg(
  754     R1, R1_H
  755 );
  756 
  757 // Class for 64 bit register r2
  758 reg_class r2_reg(
  759     R2, R2_H
  760 );
  761 
  762 // Class for 64 bit register r3
  763 reg_class r3_reg(
  764     R3, R3_H
  765 );
  766 
  767 // Class for 64 bit register r4
  768 reg_class r4_reg(
  769     R4, R4_H
  770 );
  771 
  772 // Class for 64 bit register r5
  773 reg_class r5_reg(
  774     R5, R5_H
  775 );
  776 
  777 // Class for 64 bit register r10
  778 reg_class r10_reg(
  779     R10, R10_H
  780 );
  781 
  782 // Class for 64 bit register r11
  783 reg_class r11_reg(
  784     R11, R11_H
  785 );
  786 
  787 // Class for method register
  788 reg_class method_reg(
  789     R12, R12_H
  790 );
  791 
  792 // Class for heapbase register
  793 reg_class heapbase_reg(
  794     R27, R27_H
  795 );
  796 
  797 // Class for thread register
  798 reg_class thread_reg(
  799     R28, R28_H
  800 );
  801 
  802 // Class for frame pointer register
  803 reg_class fp_reg(
  804     R29, R29_H
  805 );
  806 
  807 // Class for link register
  808 reg_class lr_reg(
  809     R30, R30_H
  810 );
  811 
  812 // Class for long sp register
  813 reg_class sp_reg(
  814   R31, R31_H
  815 );
  816 
  817 // Class for all pointer registers
  818 reg_class ptr_reg %{
  819   return _PTR_REG_mask;
  820 %}
  821 
  822 // Class for all non_special pointer registers
  823 reg_class no_special_ptr_reg %{
  824   return _NO_SPECIAL_PTR_REG_mask;
  825 %}
  826 
  827 // Class for all float registers
  828 reg_class float_reg(
  829     V0,
  830     V1,
  831     V2,
  832     V3,
  833     V4,
  834     V5,
  835     V6,
  836     V7,
  837     V8,
  838     V9,
  839     V10,
  840     V11,
  841     V12,
  842     V13,
  843     V14,
  844     V15,
  845     V16,
  846     V17,
  847     V18,
  848     V19,
  849     V20,
  850     V21,
  851     V22,
  852     V23,
  853     V24,
  854     V25,
  855     V26,
  856     V27,
  857     V28,
  858     V29,
  859     V30,
  860     V31
  861 );
  862 
  863 // Double precision float registers have virtual `high halves' that
  864 // are needed by the allocator.
  865 // Class for all double registers
  866 reg_class double_reg(
  867     V0, V0_H,
  868     V1, V1_H,
  869     V2, V2_H,
  870     V3, V3_H,
  871     V4, V4_H,
  872     V5, V5_H,
  873     V6, V6_H,
  874     V7, V7_H,
  875     V8, V8_H,
  876     V9, V9_H,
  877     V10, V10_H,
  878     V11, V11_H,
  879     V12, V12_H,
  880     V13, V13_H,
  881     V14, V14_H,
  882     V15, V15_H,
  883     V16, V16_H,
  884     V17, V17_H,
  885     V18, V18_H,
  886     V19, V19_H,
  887     V20, V20_H,
  888     V21, V21_H,
  889     V22, V22_H,
  890     V23, V23_H,
  891     V24, V24_H,
  892     V25, V25_H,
  893     V26, V26_H,
  894     V27, V27_H,
  895     V28, V28_H,
  896     V29, V29_H,
  897     V30, V30_H,
  898     V31, V31_H
  899 );
  900 
  901 // Class for all SVE vector registers.
  902 reg_class vectora_reg (
  903     V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
  904     V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
  905     V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
  906     V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
  907     V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
  908     V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
  909     V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
  910     V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
  911     V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
  912     V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
  913     V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
  914     V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
  915     V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
  916     V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
  917     V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
  918     V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
  919     V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
  920     V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
  921     V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
  922     V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
  923     V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
  924     V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
  925     V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
  926     V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
  927     V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
  928     V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
  929     V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
  930     V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
  931     V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
  932     V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
  933     V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
  934     V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
  935 );
  936 
  937 // Class for all 64bit vector registers
  938 reg_class vectord_reg(
  939     V0, V0_H,
  940     V1, V1_H,
  941     V2, V2_H,
  942     V3, V3_H,
  943     V4, V4_H,
  944     V5, V5_H,
  945     V6, V6_H,
  946     V7, V7_H,
  947     V8, V8_H,
  948     V9, V9_H,
  949     V10, V10_H,
  950     V11, V11_H,
  951     V12, V12_H,
  952     V13, V13_H,
  953     V14, V14_H,
  954     V15, V15_H,
  955     V16, V16_H,
  956     V17, V17_H,
  957     V18, V18_H,
  958     V19, V19_H,
  959     V20, V20_H,
  960     V21, V21_H,
  961     V22, V22_H,
  962     V23, V23_H,
  963     V24, V24_H,
  964     V25, V25_H,
  965     V26, V26_H,
  966     V27, V27_H,
  967     V28, V28_H,
  968     V29, V29_H,
  969     V30, V30_H,
  970     V31, V31_H
  971 );
  972 
  973 // Class for all 128bit vector registers
  974 reg_class vectorx_reg(
  975     V0, V0_H, V0_J, V0_K,
  976     V1, V1_H, V1_J, V1_K,
  977     V2, V2_H, V2_J, V2_K,
  978     V3, V3_H, V3_J, V3_K,
  979     V4, V4_H, V4_J, V4_K,
  980     V5, V5_H, V5_J, V5_K,
  981     V6, V6_H, V6_J, V6_K,
  982     V7, V7_H, V7_J, V7_K,
  983     V8, V8_H, V8_J, V8_K,
  984     V9, V9_H, V9_J, V9_K,
  985     V10, V10_H, V10_J, V10_K,
  986     V11, V11_H, V11_J, V11_K,
  987     V12, V12_H, V12_J, V12_K,
  988     V13, V13_H, V13_J, V13_K,
  989     V14, V14_H, V14_J, V14_K,
  990     V15, V15_H, V15_J, V15_K,
  991     V16, V16_H, V16_J, V16_K,
  992     V17, V17_H, V17_J, V17_K,
  993     V18, V18_H, V18_J, V18_K,
  994     V19, V19_H, V19_J, V19_K,
  995     V20, V20_H, V20_J, V20_K,
  996     V21, V21_H, V21_J, V21_K,
  997     V22, V22_H, V22_J, V22_K,
  998     V23, V23_H, V23_J, V23_K,
  999     V24, V24_H, V24_J, V24_K,
 1000     V25, V25_H, V25_J, V25_K,
 1001     V26, V26_H, V26_J, V26_K,
 1002     V27, V27_H, V27_J, V27_K,
 1003     V28, V28_H, V28_J, V28_K,
 1004     V29, V29_H, V29_J, V29_K,
 1005     V30, V30_H, V30_J, V30_K,
 1006     V31, V31_H, V31_J, V31_K
 1007 );
 1008 
 1009 // Class for 128 bit register v0
 1010 reg_class v0_reg(
 1011     V0, V0_H
 1012 );
 1013 
 1014 // Class for 128 bit register v1
 1015 reg_class v1_reg(
 1016     V1, V1_H
 1017 );
 1018 
 1019 // Class for 128 bit register v2
 1020 reg_class v2_reg(
 1021     V2, V2_H
 1022 );
 1023 
 1024 // Class for 128 bit register v3
 1025 reg_class v3_reg(
 1026     V3, V3_H
 1027 );
 1028 
 1029 // Class for 128 bit register v4
 1030 reg_class v4_reg(
 1031     V4, V4_H
 1032 );
 1033 
 1034 // Class for 128 bit register v5
 1035 reg_class v5_reg(
 1036     V5, V5_H
 1037 );
 1038 
 1039 // Class for 128 bit register v6
 1040 reg_class v6_reg(
 1041     V6, V6_H
 1042 );
 1043 
 1044 // Class for 128 bit register v7
 1045 reg_class v7_reg(
 1046     V7, V7_H
 1047 );
 1048 
 1049 // Class for 128 bit register v8
 1050 reg_class v8_reg(
 1051     V8, V8_H
 1052 );
 1053 
 1054 // Class for 128 bit register v9
 1055 reg_class v9_reg(
 1056     V9, V9_H
 1057 );
 1058 
 1059 // Class for 128 bit register v10
 1060 reg_class v10_reg(
 1061     V10, V10_H
 1062 );
 1063 
 1064 // Class for 128 bit register v11
 1065 reg_class v11_reg(
 1066     V11, V11_H
 1067 );
 1068 
 1069 // Class for 128 bit register v12
 1070 reg_class v12_reg(
 1071     V12, V12_H
 1072 );
 1073 
 1074 // Class for 128 bit register v13
 1075 reg_class v13_reg(
 1076     V13, V13_H
 1077 );
 1078 
 1079 // Class for 128 bit register v14
 1080 reg_class v14_reg(
 1081     V14, V14_H
 1082 );
 1083 
 1084 // Class for 128 bit register v15
 1085 reg_class v15_reg(
 1086     V15, V15_H
 1087 );
 1088 
 1089 // Class for 128 bit register v16
 1090 reg_class v16_reg(
 1091     V16, V16_H
 1092 );
 1093 
 1094 // Class for 128 bit register v17
 1095 reg_class v17_reg(
 1096     V17, V17_H
 1097 );
 1098 
 1099 // Class for 128 bit register v18
 1100 reg_class v18_reg(
 1101     V18, V18_H
 1102 );
 1103 
 1104 // Class for 128 bit register v19
 1105 reg_class v19_reg(
 1106     V19, V19_H
 1107 );
 1108 
 1109 // Class for 128 bit register v20
 1110 reg_class v20_reg(
 1111     V20, V20_H
 1112 );
 1113 
 1114 // Class for 128 bit register v21
 1115 reg_class v21_reg(
 1116     V21, V21_H
 1117 );
 1118 
 1119 // Class for 128 bit register v22
 1120 reg_class v22_reg(
 1121     V22, V22_H
 1122 );
 1123 
 1124 // Class for 128 bit register v23
 1125 reg_class v23_reg(
 1126     V23, V23_H
 1127 );
 1128 
 1129 // Class for 128 bit register v24
 1130 reg_class v24_reg(
 1131     V24, V24_H
 1132 );
 1133 
 1134 // Class for 128 bit register v25
 1135 reg_class v25_reg(
 1136     V25, V25_H
 1137 );
 1138 
 1139 // Class for 128 bit register v26
 1140 reg_class v26_reg(
 1141     V26, V26_H
 1142 );
 1143 
 1144 // Class for 128 bit register v27
 1145 reg_class v27_reg(
 1146     V27, V27_H
 1147 );
 1148 
 1149 // Class for 128 bit register v28
 1150 reg_class v28_reg(
 1151     V28, V28_H
 1152 );
 1153 
 1154 // Class for 128 bit register v29
 1155 reg_class v29_reg(
 1156     V29, V29_H
 1157 );
 1158 
 1159 // Class for 128 bit register v30
 1160 reg_class v30_reg(
 1161     V30, V30_H
 1162 );
 1163 
 1164 // Class for 128 bit register v31
 1165 reg_class v31_reg(
 1166     V31, V31_H
 1167 );
 1168 
 1169 // Class for all SVE predicate registers.
 1170 reg_class pr_reg (
 1171     P0,
 1172     P1,
 1173     P2,
 1174     P3,
 1175     P4,
 1176     P5,
 1177     P6,
 1178     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1179     P8,
 1180     P9,
 1181     P10,
 1182     P11,
 1183     P12,
 1184     P13,
 1185     P14,
 1186     P15
 1187 );
 1188 
 1189 // Class for SVE governing predicate registers, which are used
 1190 // to determine the active elements of a predicated instruction.
 1191 reg_class gov_pr (
 1192     P0,
 1193     P1,
 1194     P2,
 1195     P3,
 1196     P4,
 1197     P5,
 1198     P6,
 1199     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1200 );
 1201 
 1202 // Singleton class for condition codes
 1203 reg_class int_flags(RFLAGS);
 1204 
 1205 %}
 1206 
 1207 //----------DEFINITION BLOCK---------------------------------------------------
 1208 // Define name --> value mappings to inform the ADLC of an integer valued name
 1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1210 // Format:
 1211 //        int_def  <name>         ( <int_value>, <expression>);
 1212 // Generated Code in ad_<arch>.hpp
 1213 //        #define  <name>   (<expression>)
 1214 //        // value == <int_value>
 1215 // Generated code in ad_<arch>.cpp adlc_verification()
 1216 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1217 //
 1218 
 1219 // we follow the ppc-aix port in using a simple cost model which ranks
 1220 // register operations as cheap, memory ops as more expensive and
 1221 // branches as most expensive. the first two have a low as well as a
 1222 // normal cost. huge cost appears to be a way of saying don't do
 1223 // something
 1224 
 1225 definitions %{
 1226   // The default cost (of a register move instruction).
 1227   int_def INSN_COST            (    100,     100);
 1228   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1229   int_def CALL_COST            (    200,     2 * INSN_COST);
 1230   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1231 %}
 1232 
 1233 
 1234 //----------SOURCE BLOCK-------------------------------------------------------
 1235 // This is a block of C++ code which provides values, functions, and
 1236 // definitions necessary in the rest of the architecture description
 1237 
 1238 source_hpp %{
 1239 
 1240 #include "asm/macroAssembler.hpp"
 1241 #include "gc/shared/barrierSetAssembler.hpp"
 1242 #include "gc/shared/cardTable.hpp"
 1243 #include "gc/shared/cardTableBarrierSet.hpp"
 1244 #include "gc/shared/collectedHeap.hpp"
 1245 #include "opto/addnode.hpp"
 1246 #include "opto/convertnode.hpp"
 1247 #include "runtime/objectMonitor.hpp"
 1248 
 1249 extern RegMask _ANY_REG32_mask;
 1250 extern RegMask _ANY_REG_mask;
 1251 extern RegMask _PTR_REG_mask;
 1252 extern RegMask _NO_SPECIAL_REG32_mask;
 1253 extern RegMask _NO_SPECIAL_REG_mask;
 1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1255 
 1256 class CallStubImpl {
 1257 
 1258   //--------------------------------------------------------------
 1259   //---<  Used for optimization in Compile::shorten_branches  >---
 1260   //--------------------------------------------------------------
 1261 
 1262  public:
 1263   // Size of call trampoline stub.
 1264   static uint size_call_trampoline() {
 1265     return 0; // no call trampolines on this platform
 1266   }
 1267 
 1268   // number of relocations needed by a call trampoline stub
 1269   static uint reloc_call_trampoline() {
 1270     return 0; // no call trampolines on this platform
 1271   }
 1272 };
 1273 
 1274 class HandlerImpl {
 1275 
 1276  public:
 1277 
 1278   static int emit_exception_handler(CodeBuffer &cbuf);
 1279   static int emit_deopt_handler(CodeBuffer& cbuf);
 1280 
 1281   static uint size_exception_handler() {
 1282     return MacroAssembler::far_codestub_branch_size();
 1283   }
 1284 
 1285   static uint size_deopt_handler() {
 1286     // count one adr and one far branch instruction
 1287     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1288   }
 1289 };
 1290 
 1291 class Node::PD {
 1292 public:
 1293   enum NodeFlags {
 1294     _last_flag = Node::_last_flag
 1295   };
 1296 };
 1297 
 1298  bool is_CAS(int opcode, bool maybe_volatile);
 1299 
 1300   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1301 
 1302   bool unnecessary_acquire(const Node *barrier);
 1303   bool needs_acquiring_load(const Node *load);
 1304 
 1305   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1306 
 1307   bool unnecessary_release(const Node *barrier);
 1308   bool unnecessary_volatile(const Node *barrier);
 1309   bool needs_releasing_store(const Node *store);
 1310 
 1311   // predicate controlling translation of CompareAndSwapX
 1312   bool needs_acquiring_load_exclusive(const Node *load);
 1313 
 1314   // predicate controlling addressing modes
 1315   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1316 %}
 1317 
 1318 source %{
 1319 
 1320   // Derived RegMask with conditionally allocatable registers
 1321 
 1322   void PhaseOutput::pd_perform_mach_node_analysis() {
 1323   }
 1324 
 1325   int MachNode::pd_alignment_required() const {
 1326     return 1;
 1327   }
 1328 
 1329   int MachNode::compute_padding(int current_offset) const {
 1330     return 0;
 1331   }
 1332 
 1333   RegMask _ANY_REG32_mask;
 1334   RegMask _ANY_REG_mask;
 1335   RegMask _PTR_REG_mask;
 1336   RegMask _NO_SPECIAL_REG32_mask;
 1337   RegMask _NO_SPECIAL_REG_mask;
 1338   RegMask _NO_SPECIAL_PTR_REG_mask;
 1339 
 1340   void reg_mask_init() {
 1341     // We derive below RegMask(s) from the ones which are auto-generated from
 1342     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1343     // registers conditionally reserved.
 1344 
 1345     _ANY_REG32_mask = _ALL_REG32_mask;
 1346     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1347 
 1348     _ANY_REG_mask = _ALL_REG_mask;
 1349 
 1350     _PTR_REG_mask = _ALL_REG_mask;
 1351 
 1352     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1353     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1354 
 1355     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1356     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1357 
 1358     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1359     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1360 
 1361     // r27 is not allocatable when compressed oops is on and heapbase is not
 1362     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1363     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1364       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1365       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1366       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1367     }
 1368 
 1369     // r29 is not allocatable when PreserveFramePointer is on
 1370     if (PreserveFramePointer) {
 1371       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1372       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1373       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1374     }
 1375   }
 1376 
 1377   // Optimizaton of volatile gets and puts
 1378   // -------------------------------------
 1379   //
 1380   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1381   // use to implement volatile reads and writes. For a volatile read
 1382   // we simply need
 1383   //
 1384   //   ldar<x>
 1385   //
 1386   // and for a volatile write we need
 1387   //
 1388   //   stlr<x>
 1389   //
 1390   // Alternatively, we can implement them by pairing a normal
 1391   // load/store with a memory barrier. For a volatile read we need
 1392   //
 1393   //   ldr<x>
 1394   //   dmb ishld
 1395   //
 1396   // for a volatile write
 1397   //
 1398   //   dmb ish
 1399   //   str<x>
 1400   //   dmb ish
 1401   //
 1402   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1403   // sequences. These are normally translated to an instruction
 1404   // sequence like the following
 1405   //
 1406   //   dmb      ish
 1407   // retry:
 1408   //   ldxr<x>   rval raddr
 1409   //   cmp       rval rold
 1410   //   b.ne done
 1411   //   stlxr<x>  rval, rnew, rold
 1412   //   cbnz      rval retry
 1413   // done:
 1414   //   cset      r0, eq
 1415   //   dmb ishld
 1416   //
 1417   // Note that the exclusive store is already using an stlxr
 1418   // instruction. That is required to ensure visibility to other
 1419   // threads of the exclusive write (assuming it succeeds) before that
 1420   // of any subsequent writes.
 1421   //
 1422   // The following instruction sequence is an improvement on the above
 1423   //
 1424   // retry:
 1425   //   ldaxr<x>  rval raddr
 1426   //   cmp       rval rold
 1427   //   b.ne done
 1428   //   stlxr<x>  rval, rnew, rold
 1429   //   cbnz      rval retry
 1430   // done:
 1431   //   cset      r0, eq
 1432   //
 1433   // We don't need the leading dmb ish since the stlxr guarantees
 1434   // visibility of prior writes in the case that the swap is
 1435   // successful. Crucially we don't have to worry about the case where
 1436   // the swap is not successful since no valid program should be
 1437   // relying on visibility of prior changes by the attempting thread
 1438   // in the case where the CAS fails.
 1439   //
 1440   // Similarly, we don't need the trailing dmb ishld if we substitute
 1441   // an ldaxr instruction since that will provide all the guarantees we
 1442   // require regarding observation of changes made by other threads
 1443   // before any change to the CAS address observed by the load.
 1444   //
 1445   // In order to generate the desired instruction sequence we need to
 1446   // be able to identify specific 'signature' ideal graph node
 1447   // sequences which i) occur as a translation of a volatile reads or
 1448   // writes or CAS operations and ii) do not occur through any other
 1449   // translation or graph transformation. We can then provide
 1450   // alternative aldc matching rules which translate these node
 1451   // sequences to the desired machine code sequences. Selection of the
 1452   // alternative rules can be implemented by predicates which identify
 1453   // the relevant node sequences.
 1454   //
 1455   // The ideal graph generator translates a volatile read to the node
 1456   // sequence
 1457   //
 1458   //   LoadX[mo_acquire]
 1459   //   MemBarAcquire
 1460   //
 1461   // As a special case when using the compressed oops optimization we
 1462   // may also see this variant
 1463   //
 1464   //   LoadN[mo_acquire]
 1465   //   DecodeN
 1466   //   MemBarAcquire
 1467   //
 1468   // A volatile write is translated to the node sequence
 1469   //
 1470   //   MemBarRelease
 1471   //   StoreX[mo_release] {CardMark}-optional
 1472   //   MemBarVolatile
 1473   //
 1474   // n.b. the above node patterns are generated with a strict
 1475   // 'signature' configuration of input and output dependencies (see
 1476   // the predicates below for exact details). The card mark may be as
 1477   // simple as a few extra nodes or, in a few GC configurations, may
 1478   // include more complex control flow between the leading and
 1479   // trailing memory barriers. However, whatever the card mark
 1480   // configuration these signatures are unique to translated volatile
 1481   // reads/stores -- they will not appear as a result of any other
 1482   // bytecode translation or inlining nor as a consequence of
 1483   // optimizing transforms.
 1484   //
 1485   // We also want to catch inlined unsafe volatile gets and puts and
 1486   // be able to implement them using either ldar<x>/stlr<x> or some
 1487   // combination of ldr<x>/stlr<x> and dmb instructions.
 1488   //
 1489   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1490   // normal volatile put node sequence containing an extra cpuorder
 1491   // membar
 1492   //
 1493   //   MemBarRelease
 1494   //   MemBarCPUOrder
 1495   //   StoreX[mo_release] {CardMark}-optional
 1496   //   MemBarCPUOrder
 1497   //   MemBarVolatile
 1498   //
 1499   // n.b. as an aside, a cpuorder membar is not itself subject to
 1500   // matching and translation by adlc rules.  However, the rule
 1501   // predicates need to detect its presence in order to correctly
 1502   // select the desired adlc rules.
 1503   //
 1504   // Inlined unsafe volatile gets manifest as a slightly different
 1505   // node sequence to a normal volatile get because of the
 1506   // introduction of some CPUOrder memory barriers to bracket the
 1507   // Load. However, but the same basic skeleton of a LoadX feeding a
 1508   // MemBarAcquire, possibly thorugh an optional DecodeN, is still
 1509   // present
 1510   //
 1511   //   MemBarCPUOrder
 1512   //        ||       \\
 1513   //   MemBarCPUOrder LoadX[mo_acquire]
 1514   //        ||            |
 1515   //        ||       {DecodeN} optional
 1516   //        ||       /
 1517   //     MemBarAcquire
 1518   //
 1519   // In this case the acquire membar does not directly depend on the
 1520   // load. However, we can be sure that the load is generated from an
 1521   // inlined unsafe volatile get if we see it dependent on this unique
 1522   // sequence of membar nodes. Similarly, given an acquire membar we
 1523   // can know that it was added because of an inlined unsafe volatile
 1524   // get if it is fed and feeds a cpuorder membar and if its feed
 1525   // membar also feeds an acquiring load.
 1526   //
 1527   // Finally an inlined (Unsafe) CAS operation is translated to the
 1528   // following ideal graph
 1529   //
 1530   //   MemBarRelease
 1531   //   MemBarCPUOrder
 1532   //   CompareAndSwapX {CardMark}-optional
 1533   //   MemBarCPUOrder
 1534   //   MemBarAcquire
 1535   //
 1536   // So, where we can identify these volatile read and write
 1537   // signatures we can choose to plant either of the above two code
 1538   // sequences. For a volatile read we can simply plant a normal
 1539   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1540   // also choose to inhibit translation of the MemBarAcquire and
 1541   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1542   //
 1543   // When we recognise a volatile store signature we can choose to
 1544   // plant at a dmb ish as a translation for the MemBarRelease, a
 1545   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1546   // Alternatively, we can inhibit translation of the MemBarRelease
 1547   // and MemBarVolatile and instead plant a simple stlr<x>
 1548   // instruction.
 1549   //
 1550   // when we recognise a CAS signature we can choose to plant a dmb
 1551   // ish as a translation for the MemBarRelease, the conventional
 1552   // macro-instruction sequence for the CompareAndSwap node (which
 1553   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1554   // Alternatively, we can elide generation of the dmb instructions
 1555   // and plant the alternative CompareAndSwap macro-instruction
 1556   // sequence (which uses ldaxr<x>).
 1557   //
 1558   // Of course, the above only applies when we see these signature
 1559   // configurations. We still want to plant dmb instructions in any
 1560   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1561   // MemBarVolatile. For example, at the end of a constructor which
 1562   // writes final/volatile fields we will see a MemBarRelease
 1563   // instruction and this needs a 'dmb ish' lest we risk the
 1564   // constructed object being visible without making the
 1565   // final/volatile field writes visible.
 1566   //
 1567   // n.b. the translation rules below which rely on detection of the
 1568   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1569   // If we see anything other than the signature configurations we
 1570   // always just translate the loads and stores to ldr<x> and str<x>
 1571   // and translate acquire, release and volatile membars to the
 1572   // relevant dmb instructions.
 1573   //
 1574 
 1575   // is_CAS(int opcode, bool maybe_volatile)
 1576   //
 1577   // return true if opcode is one of the possible CompareAndSwapX
 1578   // values otherwise false.
 1579 
 1580   bool is_CAS(int opcode, bool maybe_volatile)
 1581   {
 1582     switch(opcode) {
 1583       // We handle these
 1584     case Op_CompareAndSwapI:
 1585     case Op_CompareAndSwapL:
 1586     case Op_CompareAndSwapP:
 1587     case Op_CompareAndSwapN:
 1588     case Op_ShenandoahCompareAndSwapP:
 1589     case Op_ShenandoahCompareAndSwapN:
 1590     case Op_CompareAndSwapB:
 1591     case Op_CompareAndSwapS:
 1592     case Op_GetAndSetI:
 1593     case Op_GetAndSetL:
 1594     case Op_GetAndSetP:
 1595     case Op_GetAndSetN:
 1596     case Op_GetAndAddI:
 1597     case Op_GetAndAddL:
 1598       return true;
 1599     case Op_CompareAndExchangeI:
 1600     case Op_CompareAndExchangeN:
 1601     case Op_CompareAndExchangeB:
 1602     case Op_CompareAndExchangeS:
 1603     case Op_CompareAndExchangeL:
 1604     case Op_CompareAndExchangeP:
 1605     case Op_WeakCompareAndSwapB:
 1606     case Op_WeakCompareAndSwapS:
 1607     case Op_WeakCompareAndSwapI:
 1608     case Op_WeakCompareAndSwapL:
 1609     case Op_WeakCompareAndSwapP:
 1610     case Op_WeakCompareAndSwapN:
 1611     case Op_ShenandoahWeakCompareAndSwapP:
 1612     case Op_ShenandoahWeakCompareAndSwapN:
 1613     case Op_ShenandoahCompareAndExchangeP:
 1614     case Op_ShenandoahCompareAndExchangeN:
 1615       return maybe_volatile;
 1616     default:
 1617       return false;
 1618     }
 1619   }
 1620 
 1621   // helper to determine the maximum number of Phi nodes we may need to
 1622   // traverse when searching from a card mark membar for the merge mem
 1623   // feeding a trailing membar or vice versa
 1624 
 1625 // predicates controlling emit of ldr<x>/ldar<x>
 1626 
 1627 bool unnecessary_acquire(const Node *barrier)
 1628 {
 1629   assert(barrier->is_MemBar(), "expecting a membar");
 1630 
 1631   MemBarNode* mb = barrier->as_MemBar();
 1632 
 1633   if (mb->trailing_load()) {
 1634     return true;
 1635   }
 1636 
 1637   if (mb->trailing_load_store()) {
 1638     Node* load_store = mb->in(MemBarNode::Precedent);
 1639     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1640     return is_CAS(load_store->Opcode(), true);
 1641   }
 1642 
 1643   return false;
 1644 }
 1645 
 1646 bool needs_acquiring_load(const Node *n)
 1647 {
 1648   assert(n->is_Load(), "expecting a load");
 1649   LoadNode *ld = n->as_Load();
 1650   return ld->is_acquire();
 1651 }
 1652 
 1653 bool unnecessary_release(const Node *n)
 1654 {
 1655   assert((n->is_MemBar() &&
 1656           n->Opcode() == Op_MemBarRelease),
 1657          "expecting a release membar");
 1658 
 1659   MemBarNode *barrier = n->as_MemBar();
 1660   if (!barrier->leading()) {
 1661     return false;
 1662   } else {
 1663     Node* trailing = barrier->trailing_membar();
 1664     MemBarNode* trailing_mb = trailing->as_MemBar();
 1665     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1666     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1667 
 1668     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1669     if (mem->is_Store()) {
 1670       assert(mem->as_Store()->is_release(), "");
 1671       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1672       return true;
 1673     } else {
 1674       assert(mem->is_LoadStore(), "");
 1675       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1676       return is_CAS(mem->Opcode(), true);
 1677     }
 1678   }
 1679   return false;
 1680 }
 1681 
 1682 bool unnecessary_volatile(const Node *n)
 1683 {
 1684   // assert n->is_MemBar();
 1685   MemBarNode *mbvol = n->as_MemBar();
 1686 
 1687   bool release = mbvol->trailing_store();
 1688   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1689 #ifdef ASSERT
 1690   if (release) {
 1691     Node* leading = mbvol->leading_membar();
 1692     assert(leading->Opcode() == Op_MemBarRelease, "");
 1693     assert(leading->as_MemBar()->leading_store(), "");
 1694     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1695   }
 1696 #endif
 1697 
 1698   return release;
 1699 }
 1700 
 1701 // predicates controlling emit of str<x>/stlr<x>
 1702 
 1703 bool needs_releasing_store(const Node *n)
 1704 {
 1705   // assert n->is_Store();
 1706   StoreNode *st = n->as_Store();
 1707   return st->trailing_membar() != NULL;
 1708 }
 1709 
 1710 // predicate controlling translation of CAS
 1711 //
 1712 // returns true if CAS needs to use an acquiring load otherwise false
 1713 
 1714 bool needs_acquiring_load_exclusive(const Node *n)
 1715 {
 1716   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1717   LoadStoreNode* ldst = n->as_LoadStore();
 1718   if (is_CAS(n->Opcode(), false)) {
 1719     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1720   } else {
 1721     return ldst->trailing_membar() != NULL;
 1722   }
 1723 
 1724   // so we can just return true here
 1725   return true;
 1726 }
 1727 
 1728 #define __ _masm.
 1729 
 1730 // advance declarations for helper functions to convert register
 1731 // indices to register objects
 1732 
 1733 // the ad file has to provide implementations of certain methods
 1734 // expected by the generic code
 1735 //
 1736 // REQUIRED FUNCTIONALITY
 1737 
 1738 //=============================================================================
 1739 
 1740 // !!!!! Special hack to get all types of calls to specify the byte offset
 1741 //       from the start of the call to the point where the return address
 1742 //       will point.
 1743 
 1744 int MachCallStaticJavaNode::ret_addr_offset()
 1745 {
 1746   // call should be a simple bl
 1747   int off = 4;
 1748   return off;
 1749 }
 1750 
 1751 int MachCallDynamicJavaNode::ret_addr_offset()
 1752 {
 1753   return 16; // movz, movk, movk, bl
 1754 }
 1755 
 1756 int MachCallRuntimeNode::ret_addr_offset() {
 1757   // for generated stubs the call will be
 1758   //   bl(addr)
 1759   // or with far branches
 1760   //   bl(trampoline_stub)
 1761   // for real runtime callouts it will be six instructions
 1762   // see aarch64_enc_java_to_runtime
 1763   //   adr(rscratch2, retaddr)
 1764   //   lea(rscratch1, RuntimeAddress(addr)
 1765   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1766   //   blr(rscratch1)
 1767   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1768   if (cb) {
 1769     return 1 * NativeInstruction::instruction_size;
 1770   } else {
 1771     return 6 * NativeInstruction::instruction_size;
 1772   }
 1773 }
 1774 
 1775 int MachCallNativeNode::ret_addr_offset() {
 1776   // This is implemented using aarch64_enc_java_to_runtime as above.
 1777   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1778   if (cb) {
 1779     return 1 * NativeInstruction::instruction_size;
 1780   } else {
 1781     return 6 * NativeInstruction::instruction_size;
 1782   }
 1783 }
 1784 
 1785 //=============================================================================
 1786 
 1787 #ifndef PRODUCT
 1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1789   st->print("BREAKPOINT");
 1790 }
 1791 #endif
 1792 
 1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1794   C2_MacroAssembler _masm(&cbuf);
 1795   __ brk(0);
 1796 }
 1797 
 1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1799   return MachNode::size(ra_);
 1800 }
 1801 
 1802 //=============================================================================
 1803 
 1804 #ifndef PRODUCT
 1805   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1806     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1807   }
 1808 #endif
 1809 
 1810   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1811     C2_MacroAssembler _masm(&cbuf);
 1812     for (int i = 0; i < _count; i++) {
 1813       __ nop();
 1814     }
 1815   }
 1816 
 1817   uint MachNopNode::size(PhaseRegAlloc*) const {
 1818     return _count * NativeInstruction::instruction_size;
 1819   }
 1820 
 1821 //=============================================================================
 1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1823 
 1824 int ConstantTable::calculate_table_base_offset() const {
 1825   return 0;  // absolute addressing, no offset
 1826 }
 1827 
 1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1830   ShouldNotReachHere();
 1831 }
 1832 
 1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1834   // Empty encoding
 1835 }
 1836 
 1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1838   return 0;
 1839 }
 1840 
 1841 #ifndef PRODUCT
 1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1843   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1844 }
 1845 #endif
 1846 
 1847 #ifndef PRODUCT
 1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1849   Compile* C = ra_->C;
 1850 
 1851   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1852 
 1853   if (C->output()->need_stack_bang(framesize))
 1854     st->print("# stack bang size=%d\n\t", framesize);
 1855 
 1856   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1857     st->print("sub  sp, sp, #%d\n\t", framesize);
 1858     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1859     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1860   } else {
 1861     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1862     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1863     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1864     st->print("sub  sp, sp, rscratch1");
 1865   }
 1866   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1867     st->print("\n\t");
 1868     st->print("ldr  rscratch1, [guard]\n\t");
 1869     st->print("dmb ishld\n\t");
 1870     st->print("ldr  rscratch2, [rthread, #thread_disarmed_offset]\n\t");
 1871     st->print("cmp  rscratch1, rscratch2\n\t");
 1872     st->print("b.eq skip");
 1873     st->print("\n\t");
 1874     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1875     st->print("b skip\n\t");
 1876     st->print("guard: int\n\t");
 1877     st->print("\n\t");
 1878     st->print("skip:\n\t");
 1879   }
 1880 }
 1881 #endif
 1882 
 1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1884   Compile* C = ra_->C;
 1885   C2_MacroAssembler _masm(&cbuf);
 1886 
 1887   // n.b. frame size includes space for return pc and rfp
 1888   const int framesize = C->output()->frame_size_in_bytes();
 1889 
 1890   // insert a nop at the start of the prolog so we can patch in a
 1891   // branch if we need to invalidate the method later
 1892   __ nop();
 1893 
 1894   if (C->clinit_barrier_on_entry()) {
 1895     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1896 
 1897     Label L_skip_barrier;
 1898 
 1899     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1900     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1901     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1902     __ bind(L_skip_barrier);
 1903   }
 1904 
 1905   if (C->max_vector_size() >= 16) {
 1906     __ reinitialize_ptrue();
 1907   }
 1908 
 1909   int bangsize = C->output()->bang_size_in_bytes();
 1910   if (C->output()->need_stack_bang(bangsize))
 1911     __ generate_stack_overflow_check(bangsize);
 1912 
 1913   __ build_frame(framesize);
 1914 
 1915   if (C->stub_function() == NULL) {
 1916     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1917     bs->nmethod_entry_barrier(&_masm);
 1918   }
 1919 
 1920   if (VerifyStackAtCalls) {
 1921     Unimplemented();
 1922   }
 1923 
 1924   C->output()->set_frame_complete(cbuf.insts_size());
 1925 
 1926   if (C->has_mach_constant_base_node()) {
 1927     // NOTE: We set the table base offset here because users might be
 1928     // emitted before MachConstantBaseNode.
 1929     ConstantTable& constant_table = C->output()->constant_table();
 1930     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1931   }
 1932 }
 1933 
 1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1935 {
 1936   return MachNode::size(ra_); // too many variables; just compute it
 1937                               // the hard way
 1938 }
 1939 
 1940 int MachPrologNode::reloc() const
 1941 {
 1942   return 0;
 1943 }
 1944 
 1945 //=============================================================================
 1946 
 1947 #ifndef PRODUCT
 1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1949   Compile* C = ra_->C;
 1950   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1951 
 1952   st->print("# pop frame %d\n\t",framesize);
 1953 
 1954   if (framesize == 0) {
 1955     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1956   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1957     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1958     st->print("add  sp, sp, #%d\n\t", framesize);
 1959   } else {
 1960     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1961     st->print("add  sp, sp, rscratch1\n\t");
 1962     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1963   }
 1964 
 1965   if (do_polling() && C->is_method_compilation()) {
 1966     st->print("# test polling word\n\t");
 1967     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1968     st->print("cmp  sp, rscratch1\n\t");
 1969     st->print("bhi #slow_path");
 1970   }
 1971 }
 1972 #endif
 1973 
 1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1975   Compile* C = ra_->C;
 1976   C2_MacroAssembler _masm(&cbuf);
 1977   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1978 
 1979   __ remove_frame(framesize);
 1980 
 1981   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1982     __ reserved_stack_check();
 1983   }
 1984 
 1985   if (do_polling() && C->is_method_compilation()) {
 1986     Label dummy_label;
 1987     Label* code_stub = &dummy_label;
 1988     if (!C->output()->in_scratch_emit_size()) {
 1989       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1990     }
 1991     __ relocate(relocInfo::poll_return_type);
 1992     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1993   }
 1994 }
 1995 
 1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1997   // Variable size. Determine dynamically.
 1998   return MachNode::size(ra_);
 1999 }
 2000 
 2001 int MachEpilogNode::reloc() const {
 2002   // Return number of relocatable values contained in this instruction.
 2003   return 1; // 1 for polling page.
 2004 }
 2005 
 2006 const Pipeline * MachEpilogNode::pipeline() const {
 2007   return MachNode::pipeline_class();
 2008 }
 2009 
 2010 //=============================================================================
 2011 
 2012 // Figure out which register class each belongs in: rc_int, rc_float or
 2013 // rc_stack.
 2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 2015 
 2016 static enum RC rc_class(OptoReg::Name reg) {
 2017 
 2018   if (reg == OptoReg::Bad) {
 2019     return rc_bad;
 2020   }
 2021 
 2022   // we have 32 int registers * 2 halves
 2023   int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
 2024 
 2025   if (reg < slots_of_int_registers) {
 2026     return rc_int;
 2027   }
 2028 
 2029   // we have 32 float register * 8 halves
 2030   int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
 2031   if (reg < slots_of_int_registers + slots_of_float_registers) {
 2032     return rc_float;
 2033   }
 2034 
 2035   int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
 2036   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 2037     return rc_predicate;
 2038   }
 2039 
 2040   // Between predicate regs & stack is the flags.
 2041   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 2042 
 2043   return rc_stack;
 2044 }
 2045 
 2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 2047   Compile* C = ra_->C;
 2048 
 2049   // Get registers to move.
 2050   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 2051   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 2052   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 2053   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 2054 
 2055   enum RC src_hi_rc = rc_class(src_hi);
 2056   enum RC src_lo_rc = rc_class(src_lo);
 2057   enum RC dst_hi_rc = rc_class(dst_hi);
 2058   enum RC dst_lo_rc = rc_class(dst_lo);
 2059 
 2060   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 2061 
 2062   if (src_hi != OptoReg::Bad) {
 2063     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 2064            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 2065            "expected aligned-adjacent pairs");
 2066   }
 2067 
 2068   if (src_lo == dst_lo && src_hi == dst_hi) {
 2069     return 0;            // Self copy, no move.
 2070   }
 2071 
 2072   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 2073               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 2074   int src_offset = ra_->reg2offset(src_lo);
 2075   int dst_offset = ra_->reg2offset(dst_lo);
 2076 
 2077   if (bottom_type()->isa_vect() != NULL) {
 2078     uint ireg = ideal_reg();
 2079     if (ireg == Op_VecA && cbuf) {
 2080       C2_MacroAssembler _masm(cbuf);
 2081       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 2082       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2083         // stack->stack
 2084         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 2085                                                 sve_vector_reg_size_in_bytes);
 2086       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2087         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2088                             sve_vector_reg_size_in_bytes);
 2089       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2090         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2091                               sve_vector_reg_size_in_bytes);
 2092       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2093         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2094                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2095                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2096       } else {
 2097         ShouldNotReachHere();
 2098       }
 2099     } else if (cbuf) {
 2100       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2101       C2_MacroAssembler _masm(cbuf);
 2102       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2103       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2104         // stack->stack
 2105         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2106         if (ireg == Op_VecD) {
 2107           __ unspill(rscratch1, true, src_offset);
 2108           __ spill(rscratch1, true, dst_offset);
 2109         } else {
 2110           __ spill_copy128(src_offset, dst_offset);
 2111         }
 2112       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2113         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2114                ireg == Op_VecD ? __ T8B : __ T16B,
 2115                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2116       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2117         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2118                  ireg == Op_VecD ? __ D : __ Q,
 2119                  ra_->reg2offset(dst_lo));
 2120       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2121         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2122                    ireg == Op_VecD ? __ D : __ Q,
 2123                    ra_->reg2offset(src_lo));
 2124       } else {
 2125         ShouldNotReachHere();
 2126       }
 2127     }
 2128   } else if (cbuf) {
 2129     C2_MacroAssembler _masm(cbuf);
 2130     switch (src_lo_rc) {
 2131     case rc_int:
 2132       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2133         if (is64) {
 2134             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2135                    as_Register(Matcher::_regEncode[src_lo]));
 2136         } else {
 2137             C2_MacroAssembler _masm(cbuf);
 2138             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2139                     as_Register(Matcher::_regEncode[src_lo]));
 2140         }
 2141       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2142         if (is64) {
 2143             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2144                      as_Register(Matcher::_regEncode[src_lo]));
 2145         } else {
 2146             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2147                      as_Register(Matcher::_regEncode[src_lo]));
 2148         }
 2149       } else {                    // gpr --> stack spill
 2150         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2151         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2152       }
 2153       break;
 2154     case rc_float:
 2155       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2156         if (is64) {
 2157             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2158                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2159         } else {
 2160             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2161                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2162         }
 2163       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2164           if (cbuf) {
 2165             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2166                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2167         } else {
 2168             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2169                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2170         }
 2171       } else {                    // fpr --> stack spill
 2172         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2173         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2174                  is64 ? __ D : __ S, dst_offset);
 2175       }
 2176       break;
 2177     case rc_stack:
 2178       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2179         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2180       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2181         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2182                    is64 ? __ D : __ S, src_offset);
 2183       } else {                    // stack --> stack copy
 2184         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2185         __ unspill(rscratch1, is64, src_offset);
 2186         __ spill(rscratch1, is64, dst_offset);
 2187       }
 2188       break;
 2189     default:
 2190       assert(false, "bad rc_class for spill");
 2191       ShouldNotReachHere();
 2192     }
 2193   }
 2194 
 2195   if (st) {
 2196     st->print("spill ");
 2197     if (src_lo_rc == rc_stack) {
 2198       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2199     } else {
 2200       st->print("%s -> ", Matcher::regName[src_lo]);
 2201     }
 2202     if (dst_lo_rc == rc_stack) {
 2203       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2204     } else {
 2205       st->print("%s", Matcher::regName[dst_lo]);
 2206     }
 2207     if (bottom_type()->isa_vect() != NULL) {
 2208       int vsize = 0;
 2209       switch (ideal_reg()) {
 2210       case Op_VecD:
 2211         vsize = 64;
 2212         break;
 2213       case Op_VecX:
 2214         vsize = 128;
 2215         break;
 2216       case Op_VecA:
 2217         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2218         break;
 2219       default:
 2220         assert(false, "bad register type for spill");
 2221         ShouldNotReachHere();
 2222       }
 2223       st->print("\t# vector spill size = %d", vsize);
 2224     } else {
 2225       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2226     }
 2227   }
 2228 
 2229   return 0;
 2230 
 2231 }
 2232 
 2233 #ifndef PRODUCT
 2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2235   if (!ra_)
 2236     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2237   else
 2238     implementation(NULL, ra_, false, st);
 2239 }
 2240 #endif
 2241 
 2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2243   implementation(&cbuf, ra_, false, NULL);
 2244 }
 2245 
 2246 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2247   return MachNode::size(ra_);
 2248 }
 2249 
 2250 //=============================================================================
 2251 
 2252 #ifndef PRODUCT
 2253 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2254   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2255   int reg = ra_->get_reg_first(this);
 2256   st->print("add %s, rsp, #%d]\t# box lock",
 2257             Matcher::regName[reg], offset);
 2258 }
 2259 #endif
 2260 
 2261 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2262   C2_MacroAssembler _masm(&cbuf);
 2263 
 2264   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2265   int reg    = ra_->get_encode(this);
 2266 
 2267   // This add will handle any 24-bit signed offset. 24 bits allows an
 2268   // 8 megabyte stack frame.
 2269   __ add(as_Register(reg), sp, offset);
 2270 }
 2271 
 2272 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2273   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2274   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2275 
 2276   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2277     return NativeInstruction::instruction_size;
 2278   } else {
 2279     return 2 * NativeInstruction::instruction_size;
 2280   }
 2281 }
 2282 
 2283 //=============================================================================
 2284 
 2285 #ifndef PRODUCT
 2286 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2287 {
 2288   st->print_cr("# MachUEPNode");
 2289   if (UseCompressedClassPointers) {
 2290     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2291     if (CompressedKlassPointers::shift() != 0) {
 2292       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2293     }
 2294   } else {
 2295    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2296   }
 2297   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2298   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2299 }
 2300 #endif
 2301 
 2302 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2303 {
 2304   // This is the unverified entry point.
 2305   C2_MacroAssembler _masm(&cbuf);
 2306 
 2307   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2308   Label skip;
 2309   // TODO
 2310   // can we avoid this skip and still use a reloc?
 2311   __ br(Assembler::EQ, skip);
 2312   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2313   __ bind(skip);
 2314 }
 2315 
 2316 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2317 {
 2318   return MachNode::size(ra_);
 2319 }
 2320 
 2321 // REQUIRED EMIT CODE
 2322 
 2323 //=============================================================================
 2324 
 2325 // Emit exception handler code.
 2326 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2327 {
 2328   // mov rscratch1 #exception_blob_entry_point
 2329   // br rscratch1
 2330   // Note that the code buffer's insts_mark is always relative to insts.
 2331   // That's why we must use the macroassembler to generate a handler.
 2332   C2_MacroAssembler _masm(&cbuf);
 2333   address base = __ start_a_stub(size_exception_handler());
 2334   if (base == NULL) {
 2335     ciEnv::current()->record_failure("CodeCache is full");
 2336     return 0;  // CodeBuffer::expand failed
 2337   }
 2338   int offset = __ offset();
 2339   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2340   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2341   __ end_a_stub();
 2342   return offset;
 2343 }
 2344 
 2345 // Emit deopt handler code.
 2346 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2347 {
 2348   // Note that the code buffer's insts_mark is always relative to insts.
 2349   // That's why we must use the macroassembler to generate a handler.
 2350   C2_MacroAssembler _masm(&cbuf);
 2351   address base = __ start_a_stub(size_deopt_handler());
 2352   if (base == NULL) {
 2353     ciEnv::current()->record_failure("CodeCache is full");
 2354     return 0;  // CodeBuffer::expand failed
 2355   }
 2356   int offset = __ offset();
 2357 
 2358   __ adr(lr, __ pc());
 2359   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2360 
 2361   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2362   __ end_a_stub();
 2363   return offset;
 2364 }
 2365 
 2366 // REQUIRED MATCHER CODE
 2367 
 2368 //=============================================================================
 2369 
 2370 const bool Matcher::match_rule_supported(int opcode) {
 2371   if (!has_match_rule(opcode))
 2372     return false;
 2373 
 2374   bool ret_value = true;
 2375   switch (opcode) {
 2376     case Op_OnSpinWait:
 2377       return VM_Version::supports_on_spin_wait();
 2378     case Op_CacheWB:
 2379     case Op_CacheWBPreSync:
 2380     case Op_CacheWBPostSync:
 2381       if (!VM_Version::supports_data_cache_line_flush()) {
 2382         ret_value = false;
 2383       }
 2384       break;
 2385   }
 2386 
 2387   return ret_value; // Per default match rules are supported.
 2388 }
 2389 
 2390 // Identify extra cases that we might want to provide match rules for vector nodes and
 2391 // other intrinsics guarded with vector length (vlen) and element type (bt).
 2392 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
 2393   if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
 2394     return false;
 2395   }
 2396   int bit_size = vlen * type2aelembytes(bt) * 8;
 2397   if (UseSVE == 0 && bit_size > 128) {
 2398     return false;
 2399   }
 2400   if (UseSVE > 0) {
 2401     return op_sve_supported(opcode);
 2402   } else { // NEON
 2403     // Special cases
 2404     switch (opcode) {
 2405     case Op_VectorMaskCmp:
 2406     // We don't have VectorReinterpret with bit_size less than 64 support for
 2407     // now, even for byte type. To be refined with fully VectorCast support.
 2408     case Op_VectorReinterpret:
 2409       if (vlen < 2 || bit_size < 64) {
 2410         return false;
 2411       }
 2412       break;
 2413     case Op_MulAddVS2VI:
 2414       if (bit_size < 128) {
 2415         return false;
 2416       }
 2417       break;
 2418     case Op_MulVL:
 2419       return false;
 2420     case Op_VectorLoadShuffle:
 2421     case Op_VectorRearrange:
 2422       if (vlen < 4) {
 2423         return false;
 2424       }
 2425       break;
 2426     // Some types of VectorCast are not implemented for now.
 2427     case Op_VectorCastI2X:
 2428       if (bt == T_BYTE) {
 2429         return false;
 2430       }
 2431       break;
 2432     case Op_VectorCastS2X:
 2433       if (vlen < 4 || bit_size < 64) {
 2434         return false;
 2435       }
 2436       break;
 2437     case Op_VectorCastF2X:
 2438     case Op_VectorCastD2X:
 2439       if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
 2440         return false;
 2441       }
 2442       break;
 2443     default:
 2444       break;
 2445     }
 2446   }
 2447   return true; // Per default match rules are supported.
 2448 }
 2449 
 2450 const RegMask* Matcher::predicate_reg_mask(void) {
 2451   return &_PR_REG_mask;
 2452 }
 2453 
 2454 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2455   return new TypeVectMask(elemTy, length);
 2456 }
 2457 
 2458 // Vector calling convention not yet implemented.
 2459 const bool Matcher::supports_vector_calling_convention(void) {
 2460   return false;
 2461 }
 2462 
 2463 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2464   Unimplemented();
 2465   return OptoRegPair(0, 0);
 2466 }
 2467 
 2468 const int Matcher::float_pressure(int default_pressure_threshold) {
 2469   return default_pressure_threshold;
 2470 }
 2471 
 2472 // Is this branch offset short enough that a short branch can be used?
 2473 //
 2474 // NOTE: If the platform does not provide any short branch variants, then
 2475 //       this method should return false for offset 0.
 2476 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2477   // The passed offset is relative to address of the branch.
 2478 
 2479   return (-32768 <= offset && offset < 32768);
 2480 }
 2481 
 2482 // Vector width in bytes.
 2483 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2484   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2485   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2486   // Minimum 2 values in vector
 2487   if (size < 2*type2aelembytes(bt)) size = 0;
 2488   // But never < 4
 2489   if (size < 4) size = 0;
 2490   return size;
 2491 }
 2492 
 2493 // Limits on vector size (number of elements) loaded into vector.
 2494 const int Matcher::max_vector_size(const BasicType bt) {
 2495   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2496 }
 2497 const int Matcher::min_vector_size(const BasicType bt) {
 2498   int max_size = max_vector_size(bt);
 2499   if ((UseSVE > 0) && (MaxVectorSize >= 16)) {
 2500     // Currently vector length less than SVE vector register size is not supported.
 2501     return max_size;
 2502   } else { // NEON
 2503     // Limit the vector size to 8 bytes
 2504     int size = 8 / type2aelembytes(bt);
 2505     if (bt == T_BYTE) {
 2506       // To support vector api shuffle/rearrange.
 2507       size = 4;
 2508     } else if (bt == T_BOOLEAN) {
 2509       // To support vector api load/store mask.
 2510       size = 2;
 2511     }
 2512     if (size < 2) size = 2;
 2513     return MIN2(size,max_size);
 2514   }
 2515 }
 2516 
 2517 // Actual max scalable vector register length.
 2518 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2519   return Matcher::max_vector_size(bt);
 2520 }
 2521 
 2522 // Vector ideal reg.
 2523 const uint Matcher::vector_ideal_reg(int len) {
 2524   if (UseSVE > 0 && 16 <= len && len <= 256) {
 2525     return Op_VecA;
 2526   }
 2527   switch(len) {
 2528     // For 16-bit/32-bit mask vector, reuse VecD.
 2529     case  2:
 2530     case  4:
 2531     case  8: return Op_VecD;
 2532     case 16: return Op_VecX;
 2533   }
 2534   ShouldNotReachHere();
 2535   return 0;
 2536 }
 2537 
 2538 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
 2539   ShouldNotReachHere(); // generic vector operands not supported
 2540   return NULL;
 2541 }
 2542 
 2543 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
 2544   ShouldNotReachHere();  // generic vector operands not supported
 2545   return false;
 2546 }
 2547 
 2548 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2549   ShouldNotReachHere();  // generic vector operands not supported
 2550   return false;
 2551 }
 2552 
 2553 // Return whether or not this register is ever used as an argument.
 2554 // This function is used on startup to build the trampoline stubs in
 2555 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2556 // call in the trampoline, and arguments in those registers not be
 2557 // available to the callee.
 2558 bool Matcher::can_be_java_arg(int reg)
 2559 {
 2560   return
 2561     reg ==  R0_num || reg == R0_H_num ||
 2562     reg ==  R1_num || reg == R1_H_num ||
 2563     reg ==  R2_num || reg == R2_H_num ||
 2564     reg ==  R3_num || reg == R3_H_num ||
 2565     reg ==  R4_num || reg == R4_H_num ||
 2566     reg ==  R5_num || reg == R5_H_num ||
 2567     reg ==  R6_num || reg == R6_H_num ||
 2568     reg ==  R7_num || reg == R7_H_num ||
 2569     reg ==  V0_num || reg == V0_H_num ||
 2570     reg ==  V1_num || reg == V1_H_num ||
 2571     reg ==  V2_num || reg == V2_H_num ||
 2572     reg ==  V3_num || reg == V3_H_num ||
 2573     reg ==  V4_num || reg == V4_H_num ||
 2574     reg ==  V5_num || reg == V5_H_num ||
 2575     reg ==  V6_num || reg == V6_H_num ||
 2576     reg ==  V7_num || reg == V7_H_num;
 2577 }
 2578 
 2579 bool Matcher::is_spillable_arg(int reg)
 2580 {
 2581   return can_be_java_arg(reg);
 2582 }
 2583 
 2584 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2585   return false;
 2586 }
 2587 
 2588 RegMask Matcher::divI_proj_mask() {
 2589   ShouldNotReachHere();
 2590   return RegMask();
 2591 }
 2592 
 2593 // Register for MODI projection of divmodI.
 2594 RegMask Matcher::modI_proj_mask() {
 2595   ShouldNotReachHere();
 2596   return RegMask();
 2597 }
 2598 
 2599 // Register for DIVL projection of divmodL.
 2600 RegMask Matcher::divL_proj_mask() {
 2601   ShouldNotReachHere();
 2602   return RegMask();
 2603 }
 2604 
 2605 // Register for MODL projection of divmodL.
 2606 RegMask Matcher::modL_proj_mask() {
 2607   ShouldNotReachHere();
 2608   return RegMask();
 2609 }
 2610 
 2611 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2612   return FP_REG_mask();
 2613 }
 2614 
 2615 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2616   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2617     Node* u = addp->fast_out(i);
 2618     if (u->is_LoadStore()) {
 2619       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2620       // instructions) only take register indirect as an operand, so
 2621       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2622       // must fail.
 2623       return false;
 2624     }
 2625     if (u->is_Mem()) {
 2626       int opsize = u->as_Mem()->memory_size();
 2627       assert(opsize > 0, "unexpected memory operand size");
 2628       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2629         return false;
 2630       }
 2631     }
 2632   }
 2633   return true;
 2634 }
 2635 
 2636 // Should the matcher clone input 'm' of node 'n'?
 2637 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2638   if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
 2639     mstack.push(m, Visit);           // m = ShiftCntV
 2640     return true;
 2641   }
 2642   return false;
 2643 }
 2644 
 2645 // Should the Matcher clone shifts on addressing modes, expecting them
 2646 // to be subsumed into complex addressing expressions or compute them
 2647 // into registers?
 2648 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2649   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2650     return true;
 2651   }
 2652 
 2653   Node *off = m->in(AddPNode::Offset);
 2654   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2655       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2656       // Are there other uses besides address expressions?
 2657       !is_visited(off)) {
 2658     address_visited.set(off->_idx); // Flag as address_visited
 2659     mstack.push(off->in(2), Visit);
 2660     Node *conv = off->in(1);
 2661     if (conv->Opcode() == Op_ConvI2L &&
 2662         // Are there other uses besides address expressions?
 2663         !is_visited(conv)) {
 2664       address_visited.set(conv->_idx); // Flag as address_visited
 2665       mstack.push(conv->in(1), Pre_Visit);
 2666     } else {
 2667       mstack.push(conv, Pre_Visit);
 2668     }
 2669     address_visited.test_set(m->_idx); // Flag as address_visited
 2670     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2671     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2672     return true;
 2673   } else if (off->Opcode() == Op_ConvI2L &&
 2674              // Are there other uses besides address expressions?
 2675              !is_visited(off)) {
 2676     address_visited.test_set(m->_idx); // Flag as address_visited
 2677     address_visited.set(off->_idx); // Flag as address_visited
 2678     mstack.push(off->in(1), Pre_Visit);
 2679     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2680     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2681     return true;
 2682   }
 2683   return false;
 2684 }
 2685 
 2686 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2687   C2_MacroAssembler _masm(&cbuf);                                       \
 2688   {                                                                     \
 2689     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2690     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2691     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2692     __ INSN(REG, as_Register(BASE));                                    \
 2693   }
 2694 
 2695 
 2696 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2697   {
 2698     Address::extend scale;
 2699 
 2700     // Hooboy, this is fugly.  We need a way to communicate to the
 2701     // encoder that the index needs to be sign extended, so we have to
 2702     // enumerate all the cases.
 2703     switch (opcode) {
 2704     case INDINDEXSCALEDI2L:
 2705     case INDINDEXSCALEDI2LN:
 2706     case INDINDEXI2L:
 2707     case INDINDEXI2LN:
 2708       scale = Address::sxtw(size);
 2709       break;
 2710     default:
 2711       scale = Address::lsl(size);
 2712     }
 2713 
 2714     if (index == -1) {
 2715       return Address(base, disp);
 2716     } else {
 2717       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2718       return Address(base, as_Register(index), scale);
 2719     }
 2720   }
 2721 
 2722 
 2723 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2724 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2725 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2726 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2727                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2728 
 2729   // Used for all non-volatile memory accesses.  The use of
 2730   // $mem->opcode() to discover whether this pattern uses sign-extended
 2731   // offsets is something of a kludge.
 2732   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2733                         Register reg, int opcode,
 2734                         Register base, int index, int scale, int disp,
 2735                         int size_in_memory)
 2736   {
 2737     Address addr = mem2address(opcode, base, index, scale, disp);
 2738     if (addr.getMode() == Address::base_plus_offset) {
 2739       /* If we get an out-of-range offset it is a bug in the compiler,
 2740          so we assert here. */
 2741       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2742              "c2 compiler bug");
 2743       /* Fix up any out-of-range offsets. */
 2744       assert_different_registers(rscratch1, base);
 2745       assert_different_registers(rscratch1, reg);
 2746       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2747     }
 2748     (masm.*insn)(reg, addr);
 2749   }
 2750 
 2751   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2752                         FloatRegister reg, int opcode,
 2753                         Register base, int index, int size, int disp,
 2754                         int size_in_memory)
 2755   {
 2756     Address::extend scale;
 2757 
 2758     switch (opcode) {
 2759     case INDINDEXSCALEDI2L:
 2760     case INDINDEXSCALEDI2LN:
 2761       scale = Address::sxtw(size);
 2762       break;
 2763     default:
 2764       scale = Address::lsl(size);
 2765     }
 2766 
 2767     if (index == -1) {
 2768       /* If we get an out-of-range offset it is a bug in the compiler,
 2769          so we assert here. */
 2770       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2771       /* Fix up any out-of-range offsets. */
 2772       assert_different_registers(rscratch1, base);
 2773       Address addr = Address(base, disp);
 2774       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2775       (masm.*insn)(reg, addr);
 2776     } else {
 2777       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2778       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2779     }
 2780   }
 2781 
 2782   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2783                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2784                         int opcode, Register base, int index, int size, int disp)
 2785   {
 2786     if (index == -1) {
 2787       (masm.*insn)(reg, T, Address(base, disp));
 2788     } else {
 2789       assert(disp == 0, "unsupported address mode");
 2790       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2791     }
 2792   }
 2793 
 2794 %}
 2795 
 2796 
 2797 
 2798 //----------ENCODING BLOCK-----------------------------------------------------
 2799 // This block specifies the encoding classes used by the compiler to
 2800 // output byte streams.  Encoding classes are parameterized macros
 2801 // used by Machine Instruction Nodes in order to generate the bit
 2802 // encoding of the instruction.  Operands specify their base encoding
 2803 // interface with the interface keyword.  There are currently
 2804 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2805 // COND_INTER.  REG_INTER causes an operand to generate a function
 2806 // which returns its register number when queried.  CONST_INTER causes
 2807 // an operand to generate a function which returns the value of the
 2808 // constant when queried.  MEMORY_INTER causes an operand to generate
 2809 // four functions which return the Base Register, the Index Register,
 2810 // the Scale Value, and the Offset Value of the operand when queried.
 2811 // COND_INTER causes an operand to generate six functions which return
 2812 // the encoding code (ie - encoding bits for the instruction)
 2813 // associated with each basic boolean condition for a conditional
 2814 // instruction.
 2815 //
 2816 // Instructions specify two basic values for encoding.  Again, a
 2817 // function is available to check if the constant displacement is an
 2818 // oop. They use the ins_encode keyword to specify their encoding
 2819 // classes (which must be a sequence of enc_class names, and their
 2820 // parameters, specified in the encoding block), and they use the
 2821 // opcode keyword to specify, in order, their primary, secondary, and
 2822 // tertiary opcode.  Only the opcode sections which a particular
 2823 // instruction needs for encoding need to be specified.
 2824 encode %{
 2825   // Build emit functions for each basic byte or larger field in the
 2826   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2827   // from C++ code in the enc_class source block.  Emit functions will
 2828   // live in the main source block for now.  In future, we can
 2829   // generalize this by adding a syntax that specifies the sizes of
 2830   // fields in an order, so that the adlc can build the emit functions
 2831   // automagically
 2832 
 2833   // catch all for unimplemented encodings
 2834   enc_class enc_unimplemented %{
 2835     C2_MacroAssembler _masm(&cbuf);
 2836     __ unimplemented("C2 catch all");
 2837   %}
 2838 
 2839   // BEGIN Non-volatile memory access
 2840 
 2841   // This encoding class is generated automatically from ad_encode.m4.
 2842   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2843   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2844     Register dst_reg = as_Register($dst$$reg);
 2845     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2846                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2847   %}
 2848 
 2849   // This encoding class is generated automatically from ad_encode.m4.
 2850   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2851   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2852     Register dst_reg = as_Register($dst$$reg);
 2853     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2854                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2855   %}
 2856 
 2857   // This encoding class is generated automatically from ad_encode.m4.
 2858   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2859   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2860     Register dst_reg = as_Register($dst$$reg);
 2861     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2862                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2863   %}
 2864 
 2865   // This encoding class is generated automatically from ad_encode.m4.
 2866   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2867   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2868     Register dst_reg = as_Register($dst$$reg);
 2869     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2870                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2871   %}
 2872 
 2873   // This encoding class is generated automatically from ad_encode.m4.
 2874   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2875   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2876     Register dst_reg = as_Register($dst$$reg);
 2877     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2878                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2879   %}
 2880 
 2881   // This encoding class is generated automatically from ad_encode.m4.
 2882   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2883   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2884     Register dst_reg = as_Register($dst$$reg);
 2885     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2886                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2887   %}
 2888 
 2889   // This encoding class is generated automatically from ad_encode.m4.
 2890   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2891   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2892     Register dst_reg = as_Register($dst$$reg);
 2893     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2894                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2895   %}
 2896 
 2897   // This encoding class is generated automatically from ad_encode.m4.
 2898   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2899   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2900     Register dst_reg = as_Register($dst$$reg);
 2901     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2902                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2903   %}
 2904 
 2905   // This encoding class is generated automatically from ad_encode.m4.
 2906   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2907   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2908     Register dst_reg = as_Register($dst$$reg);
 2909     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2910                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2911   %}
 2912 
 2913   // This encoding class is generated automatically from ad_encode.m4.
 2914   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2915   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2916     Register dst_reg = as_Register($dst$$reg);
 2917     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2918                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2919   %}
 2920 
 2921   // This encoding class is generated automatically from ad_encode.m4.
 2922   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2923   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2924     Register dst_reg = as_Register($dst$$reg);
 2925     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2926                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2927   %}
 2928 
 2929   // This encoding class is generated automatically from ad_encode.m4.
 2930   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2931   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2932     Register dst_reg = as_Register($dst$$reg);
 2933     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2934                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2935   %}
 2936 
 2937   // This encoding class is generated automatically from ad_encode.m4.
 2938   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2939   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2940     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2941     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2942                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2943   %}
 2944 
 2945   // This encoding class is generated automatically from ad_encode.m4.
 2946   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2947   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2948     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2949     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2950                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2951   %}
 2952 
 2953   // This encoding class is generated automatically from ad_encode.m4.
 2954   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2955   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2956     Register src_reg = as_Register($src$$reg);
 2957     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2958                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2959   %}
 2960 
 2961   // This encoding class is generated automatically from ad_encode.m4.
 2962   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2963   enc_class aarch64_enc_strb0(memory1 mem) %{
 2964     C2_MacroAssembler _masm(&cbuf);
 2965     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2967   %}
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2972     Register src_reg = as_Register($src$$reg);
 2973     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_strh0(memory2 mem) %{
 2980     C2_MacroAssembler _masm(&cbuf);
 2981     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2988     Register src_reg = as_Register($src$$reg);
 2989     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2991   %}
 2992 
 2993   // This encoding class is generated automatically from ad_encode.m4.
 2994   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2995   enc_class aarch64_enc_strw0(memory4 mem) %{
 2996     C2_MacroAssembler _masm(&cbuf);
 2997     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2998                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2999   %}
 3000 
 3001   // This encoding class is generated automatically from ad_encode.m4.
 3002   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3003   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3004     Register src_reg = as_Register($src$$reg);
 3005     // we sometimes get asked to store the stack pointer into the
 3006     // current thread -- we cannot do that directly on AArch64
 3007     if (src_reg == r31_sp) {
 3008       C2_MacroAssembler _masm(&cbuf);
 3009       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3010       __ mov(rscratch2, sp);
 3011       src_reg = rscratch2;
 3012     }
 3013     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3014                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3015   %}
 3016 
 3017   // This encoding class is generated automatically from ad_encode.m4.
 3018   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3019   enc_class aarch64_enc_str0(memory8 mem) %{
 3020     C2_MacroAssembler _masm(&cbuf);
 3021     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3022                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3023   %}
 3024 
 3025   // This encoding class is generated automatically from ad_encode.m4.
 3026   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3027   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3028     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3029     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3030                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3031   %}
 3032 
 3033   // This encoding class is generated automatically from ad_encode.m4.
 3034   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3035   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3036     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3037     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3038                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3039   %}
 3040 
 3041   // This encoding class is generated automatically from ad_encode.m4.
 3042   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3043   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3044       C2_MacroAssembler _masm(&cbuf);
 3045       __ membar(Assembler::StoreStore);
 3046       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3047                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3048   %}
 3049 
 3050   // END Non-volatile memory access
 3051 
 3052   // Vector loads and stores
 3053   enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
 3054     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3055     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3056        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3057   %}
 3058 
 3059   enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
 3060     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3061     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3062        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3063   %}
 3064 
 3065   enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
 3066     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3067     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3068        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3069   %}
 3070 
 3071   enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
 3072     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3073     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3074        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3075   %}
 3076 
 3077   enc_class aarch64_enc_strvH(vecD src, memory mem) %{
 3078     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3079     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3080        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3081   %}
 3082 
 3083   enc_class aarch64_enc_strvS(vecD src, memory mem) %{
 3084     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3085     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3086        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3087   %}
 3088 
 3089   enc_class aarch64_enc_strvD(vecD src, memory mem) %{
 3090     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3091     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3092        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3093   %}
 3094 
 3095   enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
 3096     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3097     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3098        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3099   %}
 3100 
 3101   // volatile loads and stores
 3102 
 3103   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3104     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3105                  rscratch1, stlrb);
 3106   %}
 3107 
 3108   enc_class aarch64_enc_stlrb0(memory mem) %{
 3109     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3110                  rscratch1, stlrb);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3114     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3115                  rscratch1, stlrh);
 3116   %}
 3117 
 3118   enc_class aarch64_enc_stlrh0(memory mem) %{
 3119     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3120                  rscratch1, stlrh);
 3121   %}
 3122 
 3123   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3124     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3125                  rscratch1, stlrw);
 3126   %}
 3127 
 3128   enc_class aarch64_enc_stlrw0(memory mem) %{
 3129     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3130                  rscratch1, stlrw);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3134     Register dst_reg = as_Register($dst$$reg);
 3135     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136              rscratch1, ldarb);
 3137     __ sxtbw(dst_reg, dst_reg);
 3138   %}
 3139 
 3140   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3141     Register dst_reg = as_Register($dst$$reg);
 3142     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3143              rscratch1, ldarb);
 3144     __ sxtb(dst_reg, dst_reg);
 3145   %}
 3146 
 3147   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3148     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3149              rscratch1, ldarb);
 3150   %}
 3151 
 3152   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3153     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3154              rscratch1, ldarb);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3158     Register dst_reg = as_Register($dst$$reg);
 3159     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160              rscratch1, ldarh);
 3161     __ sxthw(dst_reg, dst_reg);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3165     Register dst_reg = as_Register($dst$$reg);
 3166     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3167              rscratch1, ldarh);
 3168     __ sxth(dst_reg, dst_reg);
 3169   %}
 3170 
 3171   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3172     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3173              rscratch1, ldarh);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3177     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3178              rscratch1, ldarh);
 3179   %}
 3180 
 3181   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3182     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3183              rscratch1, ldarw);
 3184   %}
 3185 
 3186   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3187     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3188              rscratch1, ldarw);
 3189   %}
 3190 
 3191   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3192     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3193              rscratch1, ldar);
 3194   %}
 3195 
 3196   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3197     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3198              rscratch1, ldarw);
 3199     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3200   %}
 3201 
 3202   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3203     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3204              rscratch1, ldar);
 3205     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3209     Register src_reg = as_Register($src$$reg);
 3210     // we sometimes get asked to store the stack pointer into the
 3211     // current thread -- we cannot do that directly on AArch64
 3212     if (src_reg == r31_sp) {
 3213       C2_MacroAssembler _masm(&cbuf);
 3214       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3215       __ mov(rscratch2, sp);
 3216       src_reg = rscratch2;
 3217     }
 3218     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3219                  rscratch1, stlr);
 3220   %}
 3221 
 3222   enc_class aarch64_enc_stlr0(memory mem) %{
 3223     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3224                  rscratch1, stlr);
 3225   %}
 3226 
 3227   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3228     {
 3229       C2_MacroAssembler _masm(&cbuf);
 3230       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3231       __ fmovs(rscratch2, src_reg);
 3232     }
 3233     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3234                  rscratch1, stlrw);
 3235   %}
 3236 
 3237   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3238     {
 3239       C2_MacroAssembler _masm(&cbuf);
 3240       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3241       __ fmovd(rscratch2, src_reg);
 3242     }
 3243     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3244                  rscratch1, stlr);
 3245   %}
 3246 
 3247   // synchronized read/update encodings
 3248 
 3249   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3250     C2_MacroAssembler _masm(&cbuf);
 3251     Register dst_reg = as_Register($dst$$reg);
 3252     Register base = as_Register($mem$$base);
 3253     int index = $mem$$index;
 3254     int scale = $mem$$scale;
 3255     int disp = $mem$$disp;
 3256     if (index == -1) {
 3257        if (disp != 0) {
 3258         __ lea(rscratch1, Address(base, disp));
 3259         __ ldaxr(dst_reg, rscratch1);
 3260       } else {
 3261         // TODO
 3262         // should we ever get anything other than this case?
 3263         __ ldaxr(dst_reg, base);
 3264       }
 3265     } else {
 3266       Register index_reg = as_Register(index);
 3267       if (disp == 0) {
 3268         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3269         __ ldaxr(dst_reg, rscratch1);
 3270       } else {
 3271         __ lea(rscratch1, Address(base, disp));
 3272         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3273         __ ldaxr(dst_reg, rscratch1);
 3274       }
 3275     }
 3276   %}
 3277 
 3278   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3279     C2_MacroAssembler _masm(&cbuf);
 3280     Register src_reg = as_Register($src$$reg);
 3281     Register base = as_Register($mem$$base);
 3282     int index = $mem$$index;
 3283     int scale = $mem$$scale;
 3284     int disp = $mem$$disp;
 3285     if (index == -1) {
 3286        if (disp != 0) {
 3287         __ lea(rscratch2, Address(base, disp));
 3288         __ stlxr(rscratch1, src_reg, rscratch2);
 3289       } else {
 3290         // TODO
 3291         // should we ever get anything other than this case?
 3292         __ stlxr(rscratch1, src_reg, base);
 3293       }
 3294     } else {
 3295       Register index_reg = as_Register(index);
 3296       if (disp == 0) {
 3297         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3298         __ stlxr(rscratch1, src_reg, rscratch2);
 3299       } else {
 3300         __ lea(rscratch2, Address(base, disp));
 3301         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3302         __ stlxr(rscratch1, src_reg, rscratch2);
 3303       }
 3304     }
 3305     __ cmpw(rscratch1, zr);
 3306   %}
 3307 
 3308   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3309     C2_MacroAssembler _masm(&cbuf);
 3310     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3311     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3312                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3313                /*weak*/ false, noreg);
 3314   %}
 3315 
 3316   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3317     C2_MacroAssembler _masm(&cbuf);
 3318     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3319     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3320                Assembler::word, /*acquire*/ false, /*release*/ true,
 3321                /*weak*/ false, noreg);
 3322   %}
 3323 
 3324   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3325     C2_MacroAssembler _masm(&cbuf);
 3326     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3327     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3328                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3329                /*weak*/ false, noreg);
 3330   %}
 3331 
 3332   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3333     C2_MacroAssembler _masm(&cbuf);
 3334     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3335     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3336                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3337                /*weak*/ false, noreg);
 3338   %}
 3339 
 3340 
 3341   // The only difference between aarch64_enc_cmpxchg and
 3342   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3343   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3344   // lock.
 3345   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3346     C2_MacroAssembler _masm(&cbuf);
 3347     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3348     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3349                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3350                /*weak*/ false, noreg);
 3351   %}
 3352 
 3353   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3354     C2_MacroAssembler _masm(&cbuf);
 3355     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3356     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3357                Assembler::word, /*acquire*/ true, /*release*/ true,
 3358                /*weak*/ false, noreg);
 3359   %}
 3360 
 3361   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3362     C2_MacroAssembler _masm(&cbuf);
 3363     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3364     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3365                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3366                /*weak*/ false, noreg);
 3367   %}
 3368 
 3369   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3370     C2_MacroAssembler _masm(&cbuf);
 3371     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3372     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3373                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3374                /*weak*/ false, noreg);
 3375   %}
 3376 
 3377   // auxiliary used for CompareAndSwapX to set result register
 3378   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3379     C2_MacroAssembler _masm(&cbuf);
 3380     Register res_reg = as_Register($res$$reg);
 3381     __ cset(res_reg, Assembler::EQ);
 3382   %}
 3383 
 3384   // prefetch encodings
 3385 
 3386   enc_class aarch64_enc_prefetchw(memory mem) %{
 3387     C2_MacroAssembler _masm(&cbuf);
 3388     Register base = as_Register($mem$$base);
 3389     int index = $mem$$index;
 3390     int scale = $mem$$scale;
 3391     int disp = $mem$$disp;
 3392     if (index == -1) {
 3393       __ prfm(Address(base, disp), PSTL1KEEP);
 3394     } else {
 3395       Register index_reg = as_Register(index);
 3396       if (disp == 0) {
 3397         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3398       } else {
 3399         __ lea(rscratch1, Address(base, disp));
 3400 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3401       }
 3402     }
 3403   %}
 3404 
 3405   /// mov envcodings
 3406 
 3407   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3408     C2_MacroAssembler _masm(&cbuf);
 3409     uint32_t con = (uint32_t)$src$$constant;
 3410     Register dst_reg = as_Register($dst$$reg);
 3411     if (con == 0) {
 3412       __ movw(dst_reg, zr);
 3413     } else {
 3414       __ movw(dst_reg, con);
 3415     }
 3416   %}
 3417 
 3418   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3419     C2_MacroAssembler _masm(&cbuf);
 3420     Register dst_reg = as_Register($dst$$reg);
 3421     uint64_t con = (uint64_t)$src$$constant;
 3422     if (con == 0) {
 3423       __ mov(dst_reg, zr);
 3424     } else {
 3425       __ mov(dst_reg, con);
 3426     }
 3427   %}
 3428 
 3429   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3430     C2_MacroAssembler _masm(&cbuf);
 3431     Register dst_reg = as_Register($dst$$reg);
 3432     address con = (address)$src$$constant;
 3433     if (con == NULL || con == (address)1) {
 3434       ShouldNotReachHere();
 3435     } else {
 3436       relocInfo::relocType rtype = $src->constant_reloc();
 3437       if (rtype == relocInfo::oop_type) {
 3438         __ movoop(dst_reg, (jobject)con, /*immediate*/true);
 3439       } else if (rtype == relocInfo::metadata_type) {
 3440         __ mov_metadata(dst_reg, (Metadata*)con);
 3441       } else {
 3442         assert(rtype == relocInfo::none, "unexpected reloc type");
 3443         if (! __ is_valid_AArch64_address(con) ||
 3444             con < (address)(uintptr_t)os::vm_page_size()) {
 3445           __ mov(dst_reg, con);
 3446         } else {
 3447           uint64_t offset;
 3448           __ adrp(dst_reg, con, offset);
 3449           __ add(dst_reg, dst_reg, offset);
 3450         }
 3451       }
 3452     }
 3453   %}
 3454 
 3455   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3456     C2_MacroAssembler _masm(&cbuf);
 3457     Register dst_reg = as_Register($dst$$reg);
 3458     __ mov(dst_reg, zr);
 3459   %}
 3460 
 3461   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register dst_reg = as_Register($dst$$reg);
 3464     __ mov(dst_reg, (uint64_t)1);
 3465   %}
 3466 
 3467   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3468     C2_MacroAssembler _masm(&cbuf);
 3469     __ load_byte_map_base($dst$$Register);
 3470   %}
 3471 
 3472   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3473     C2_MacroAssembler _masm(&cbuf);
 3474     Register dst_reg = as_Register($dst$$reg);
 3475     address con = (address)$src$$constant;
 3476     if (con == NULL) {
 3477       ShouldNotReachHere();
 3478     } else {
 3479       relocInfo::relocType rtype = $src->constant_reloc();
 3480       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3481       __ set_narrow_oop(dst_reg, (jobject)con);
 3482     }
 3483   %}
 3484 
 3485   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3486     C2_MacroAssembler _masm(&cbuf);
 3487     Register dst_reg = as_Register($dst$$reg);
 3488     __ mov(dst_reg, zr);
 3489   %}
 3490 
 3491   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3492     C2_MacroAssembler _masm(&cbuf);
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     address con = (address)$src$$constant;
 3495     if (con == NULL) {
 3496       ShouldNotReachHere();
 3497     } else {
 3498       relocInfo::relocType rtype = $src->constant_reloc();
 3499       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3500       __ set_narrow_klass(dst_reg, (Klass *)con);
 3501     }
 3502   %}
 3503 
 3504   // arithmetic encodings
 3505 
 3506   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3507     C2_MacroAssembler _masm(&cbuf);
 3508     Register dst_reg = as_Register($dst$$reg);
 3509     Register src_reg = as_Register($src1$$reg);
 3510     int32_t con = (int32_t)$src2$$constant;
 3511     // add has primary == 0, subtract has primary == 1
 3512     if ($primary) { con = -con; }
 3513     if (con < 0) {
 3514       __ subw(dst_reg, src_reg, -con);
 3515     } else {
 3516       __ addw(dst_reg, src_reg, con);
 3517     }
 3518   %}
 3519 
 3520   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3521     C2_MacroAssembler _masm(&cbuf);
 3522     Register dst_reg = as_Register($dst$$reg);
 3523     Register src_reg = as_Register($src1$$reg);
 3524     int32_t con = (int32_t)$src2$$constant;
 3525     // add has primary == 0, subtract has primary == 1
 3526     if ($primary) { con = -con; }
 3527     if (con < 0) {
 3528       __ sub(dst_reg, src_reg, -con);
 3529     } else {
 3530       __ add(dst_reg, src_reg, con);
 3531     }
 3532   %}
 3533 
 3534   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3535     C2_MacroAssembler _masm(&cbuf);
 3536    Register dst_reg = as_Register($dst$$reg);
 3537    Register src1_reg = as_Register($src1$$reg);
 3538    Register src2_reg = as_Register($src2$$reg);
 3539     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3540   %}
 3541 
 3542   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3543     C2_MacroAssembler _masm(&cbuf);
 3544    Register dst_reg = as_Register($dst$$reg);
 3545    Register src1_reg = as_Register($src1$$reg);
 3546    Register src2_reg = as_Register($src2$$reg);
 3547     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3548   %}
 3549 
 3550   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3551     C2_MacroAssembler _masm(&cbuf);
 3552    Register dst_reg = as_Register($dst$$reg);
 3553    Register src1_reg = as_Register($src1$$reg);
 3554    Register src2_reg = as_Register($src2$$reg);
 3555     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3556   %}
 3557 
 3558   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3559     C2_MacroAssembler _masm(&cbuf);
 3560    Register dst_reg = as_Register($dst$$reg);
 3561    Register src1_reg = as_Register($src1$$reg);
 3562    Register src2_reg = as_Register($src2$$reg);
 3563     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3564   %}
 3565 
 3566   // compare instruction encodings
 3567 
 3568   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3569     C2_MacroAssembler _masm(&cbuf);
 3570     Register reg1 = as_Register($src1$$reg);
 3571     Register reg2 = as_Register($src2$$reg);
 3572     __ cmpw(reg1, reg2);
 3573   %}
 3574 
 3575   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3576     C2_MacroAssembler _masm(&cbuf);
 3577     Register reg = as_Register($src1$$reg);
 3578     int32_t val = $src2$$constant;
 3579     if (val >= 0) {
 3580       __ subsw(zr, reg, val);
 3581     } else {
 3582       __ addsw(zr, reg, -val);
 3583     }
 3584   %}
 3585 
 3586   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3587     C2_MacroAssembler _masm(&cbuf);
 3588     Register reg1 = as_Register($src1$$reg);
 3589     uint32_t val = (uint32_t)$src2$$constant;
 3590     __ movw(rscratch1, val);
 3591     __ cmpw(reg1, rscratch1);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3595     C2_MacroAssembler _masm(&cbuf);
 3596     Register reg1 = as_Register($src1$$reg);
 3597     Register reg2 = as_Register($src2$$reg);
 3598     __ cmp(reg1, reg2);
 3599   %}
 3600 
 3601   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3602     C2_MacroAssembler _masm(&cbuf);
 3603     Register reg = as_Register($src1$$reg);
 3604     int64_t val = $src2$$constant;
 3605     if (val >= 0) {
 3606       __ subs(zr, reg, val);
 3607     } else if (val != -val) {
 3608       __ adds(zr, reg, -val);
 3609     } else {
 3610     // aargh, Long.MIN_VALUE is a special case
 3611       __ orr(rscratch1, zr, (uint64_t)val);
 3612       __ subs(zr, reg, rscratch1);
 3613     }
 3614   %}
 3615 
 3616   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3617     C2_MacroAssembler _masm(&cbuf);
 3618     Register reg1 = as_Register($src1$$reg);
 3619     uint64_t val = (uint64_t)$src2$$constant;
 3620     __ mov(rscratch1, val);
 3621     __ cmp(reg1, rscratch1);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3625     C2_MacroAssembler _masm(&cbuf);
 3626     Register reg1 = as_Register($src1$$reg);
 3627     Register reg2 = as_Register($src2$$reg);
 3628     __ cmp(reg1, reg2);
 3629   %}
 3630 
 3631   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3632     C2_MacroAssembler _masm(&cbuf);
 3633     Register reg1 = as_Register($src1$$reg);
 3634     Register reg2 = as_Register($src2$$reg);
 3635     __ cmpw(reg1, reg2);
 3636   %}
 3637 
 3638   enc_class aarch64_enc_testp(iRegP src) %{
 3639     C2_MacroAssembler _masm(&cbuf);
 3640     Register reg = as_Register($src$$reg);
 3641     __ cmp(reg, zr);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_testn(iRegN src) %{
 3645     C2_MacroAssembler _masm(&cbuf);
 3646     Register reg = as_Register($src$$reg);
 3647     __ cmpw(reg, zr);
 3648   %}
 3649 
 3650   enc_class aarch64_enc_b(label lbl) %{
 3651     C2_MacroAssembler _masm(&cbuf);
 3652     Label *L = $lbl$$label;
 3653     __ b(*L);
 3654   %}
 3655 
 3656   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3657     C2_MacroAssembler _masm(&cbuf);
 3658     Label *L = $lbl$$label;
 3659     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3660   %}
 3661 
 3662   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3663     C2_MacroAssembler _masm(&cbuf);
 3664     Label *L = $lbl$$label;
 3665     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3666   %}
 3667 
 3668   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3669   %{
 3670      Register sub_reg = as_Register($sub$$reg);
 3671      Register super_reg = as_Register($super$$reg);
 3672      Register temp_reg = as_Register($temp$$reg);
 3673      Register result_reg = as_Register($result$$reg);
 3674 
 3675      Label miss;
 3676      C2_MacroAssembler _masm(&cbuf);
 3677      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3678                                      NULL, &miss,
 3679                                      /*set_cond_codes:*/ true);
 3680      if ($primary) {
 3681        __ mov(result_reg, zr);
 3682      }
 3683      __ bind(miss);
 3684   %}
 3685 
 3686   enc_class aarch64_enc_java_static_call(method meth) %{
 3687     C2_MacroAssembler _masm(&cbuf);
 3688 
 3689     address addr = (address)$meth$$method;
 3690     address call;
 3691     if (!_method) {
 3692       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3693       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
 3694       if (call == NULL) {
 3695         ciEnv::current()->record_failure("CodeCache is full");
 3696         return;
 3697       }
 3698     } else {
 3699       int method_index = resolved_method_index(cbuf);
 3700       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3701                                                   : static_call_Relocation::spec(method_index);
 3702       call = __ trampoline_call(Address(addr, rspec), &cbuf);
 3703       if (call == NULL) {
 3704         ciEnv::current()->record_failure("CodeCache is full");
 3705         return;
 3706       }
 3707       // Emit stub for static call
 3708       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
 3709       if (stub == NULL) {
 3710         ciEnv::current()->record_failure("CodeCache is full");
 3711         return;
 3712       }
 3713     }
 3714 
 3715     // Only non uncommon_trap calls need to reinitialize ptrue.
 3716     if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
 3717       __ reinitialize_ptrue();
 3718     }
 3719   %}
 3720 
 3721   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3722     C2_MacroAssembler _masm(&cbuf);
 3723     int method_index = resolved_method_index(cbuf);
 3724     address call = __ ic_call((address)$meth$$method, method_index);
 3725     if (call == NULL) {
 3726       ciEnv::current()->record_failure("CodeCache is full");
 3727       return;
 3728     } else if (Compile::current()->max_vector_size() >= 16) {
 3729       __ reinitialize_ptrue();
 3730     }
 3731   %}
 3732 
 3733   enc_class aarch64_enc_call_epilog() %{
 3734     C2_MacroAssembler _masm(&cbuf);
 3735     if (VerifyStackAtCalls) {
 3736       // Check that stack depth is unchanged: find majik cookie on stack
 3737       __ call_Unimplemented();
 3738     }
 3739   %}
 3740 
 3741   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3742     C2_MacroAssembler _masm(&cbuf);
 3743 
 3744     // some calls to generated routines (arraycopy code) are scheduled
 3745     // by C2 as runtime calls. if so we can call them using a br (they
 3746     // will be in a reachable segment) otherwise we have to use a blr
 3747     // which loads the absolute address into a register.
 3748     address entry = (address)$meth$$method;
 3749     CodeBlob *cb = CodeCache::find_blob(entry);
 3750     if (cb) {
 3751       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3752       if (call == NULL) {
 3753         ciEnv::current()->record_failure("CodeCache is full");
 3754         return;
 3755       }
 3756     } else {
 3757       Label retaddr;
 3758       __ adr(rscratch2, retaddr);
 3759       __ lea(rscratch1, RuntimeAddress(entry));
 3760       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3761       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3762       __ blr(rscratch1);
 3763       __ bind(retaddr);
 3764       __ add(sp, sp, 2 * wordSize);
 3765     }
 3766     if (Compile::current()->max_vector_size() >= 16) {
 3767       __ reinitialize_ptrue();
 3768     }
 3769   %}
 3770 
 3771   enc_class aarch64_enc_rethrow() %{
 3772     C2_MacroAssembler _masm(&cbuf);
 3773     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3774   %}
 3775 
 3776   enc_class aarch64_enc_ret() %{
 3777     C2_MacroAssembler _masm(&cbuf);
 3778 #ifdef ASSERT
 3779     if (Compile::current()->max_vector_size() >= 16) {
 3780       __ verify_ptrue();
 3781     }
 3782 #endif
 3783     __ ret(lr);
 3784   %}
 3785 
 3786   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3787     C2_MacroAssembler _masm(&cbuf);
 3788     Register target_reg = as_Register($jump_target$$reg);
 3789     __ br(target_reg);
 3790   %}
 3791 
 3792   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3793     C2_MacroAssembler _masm(&cbuf);
 3794     Register target_reg = as_Register($jump_target$$reg);
 3795     // exception oop should be in r0
 3796     // ret addr has been popped into lr
 3797     // callee expects it in r3
 3798     __ mov(r3, lr);
 3799     __ br(target_reg);
 3800   %}
 3801 
 3802   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3803     C2_MacroAssembler _masm(&cbuf);
 3804     Register oop = as_Register($object$$reg);
 3805     Register box = as_Register($box$$reg);
 3806     Register disp_hdr = as_Register($tmp$$reg);
 3807     Register tmp = as_Register($tmp2$$reg);
 3808     Label cont;
 3809     Label object_has_monitor;
 3810     Label cas_failed;
 3811 
 3812     assert_different_registers(oop, box, tmp, disp_hdr);
 3813 
 3814     // Load markWord from object into displaced_header.
 3815     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3816 
 3817     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3818       __ load_klass(tmp, oop);
 3819       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3820       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3821       __ br(Assembler::NE, cont);
 3822     }
 3823 
 3824     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3825       __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
 3826     }
 3827 
 3828     // Check for existing monitor
 3829     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3830 
 3831     // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3832     __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3833 
 3834     // Initialize the box. (Must happen before we update the object mark!)
 3835     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3836 
 3837     // Compare object markWord with an unlocked value (tmp) and if
 3838     // equal exchange the stack address of our box with object markWord.
 3839     // On failure disp_hdr contains the possibly locked markWord.
 3840     __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3841                /*release*/ true, /*weak*/ false, disp_hdr);
 3842     __ br(Assembler::EQ, cont);
 3843 
 3844     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3845 
 3846     // If the compare-and-exchange succeeded, then we found an unlocked
 3847     // object, will have now locked it will continue at label cont
 3848 
 3849     __ bind(cas_failed);
 3850     // We did not see an unlocked object so try the fast recursive case.
 3851 
 3852     // Check if the owner is self by comparing the value in the
 3853     // markWord of object (disp_hdr) with the stack pointer.
 3854     __ mov(rscratch1, sp);
 3855     __ sub(disp_hdr, disp_hdr, rscratch1);
 3856     __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3857     // If condition is true we are cont and hence we can store 0 as the
 3858     // displaced header in the box, which indicates that it is a recursive lock.
 3859     __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3860     __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3861 
 3862     __ b(cont);
 3863 
 3864     // Handle existing monitor.
 3865     __ bind(object_has_monitor);
 3866 
 3867     // The object's monitor m is unlocked iff m->owner == NULL,
 3868     // otherwise m->owner may contain a thread or a stack address.
 3869     //
 3870     // Try to CAS m->owner from NULL to current thread.
 3871     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3872     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3873                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3874 
 3875     // Store a non-null value into the box to avoid looking like a re-entrant
 3876     // lock. The fast-path monitor unlock code checks for
 3877     // markWord::monitor_value so use markWord::unused_mark which has the
 3878     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3879     __ mov(tmp, (address)markWord::unused_mark().value());
 3880     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3881 
 3882     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3883 
 3884     __ cmp(rscratch1, rthread);
 3885     __ br(Assembler::NE, cont); // Check for recursive locking
 3886 
 3887     // Recursive lock case
 3888     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3889     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3890 
 3891     __ bind(cont);
 3892     // flag == EQ indicates success
 3893     // flag == NE indicates failure
 3894   %}
 3895 
 3896   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3897     C2_MacroAssembler _masm(&cbuf);
 3898     Register oop = as_Register($object$$reg);
 3899     Register box = as_Register($box$$reg);
 3900     Register disp_hdr = as_Register($tmp$$reg);
 3901     Register tmp = as_Register($tmp2$$reg);
 3902     Label cont;
 3903     Label object_has_monitor;
 3904 
 3905     assert_different_registers(oop, box, tmp, disp_hdr);
 3906 
 3907     if (UseBiasedLocking && !UseOptoBiasInlining) {
 3908       __ biased_locking_exit(oop, tmp, cont);
 3909     }
 3910 
 3911     // Find the lock address and load the displaced header from the stack.
 3912     __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3913 
 3914     // If the displaced header is 0, we have a recursive unlock.
 3915     __ cmp(disp_hdr, zr);
 3916     __ br(Assembler::EQ, cont);
 3917 
 3918     // Handle existing monitor.
 3919     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3920     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3921 
 3922     // Check if it is still a light weight lock, this is is true if we
 3923     // see the stack address of the basicLock in the markWord of the
 3924     // object.
 3925 
 3926     __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3927                /*release*/ true, /*weak*/ false, tmp);
 3928     __ b(cont);
 3929 
 3930     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3931 
 3932     // Handle existing monitor.
 3933     __ bind(object_has_monitor);
 3934     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3935     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3936     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3937 
 3938     Label notRecursive;
 3939     __ cbz(disp_hdr, notRecursive);
 3940 
 3941     // Recursive lock
 3942     __ sub(disp_hdr, disp_hdr, 1u);
 3943     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3944     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3945     __ b(cont);
 3946 
 3947     __ bind(notRecursive);
 3948     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3949     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3950     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3951     __ cmp(rscratch1, zr); // Sets flags for result
 3952     __ cbnz(rscratch1, cont);
 3953     // need a release store here
 3954     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3955     __ stlr(zr, tmp); // set unowned
 3956 
 3957     __ bind(cont);
 3958     // flag == EQ indicates success
 3959     // flag == NE indicates failure
 3960   %}
 3961 
 3962 %}
 3963 
 3964 //----------FRAME--------------------------------------------------------------
 3965 // Definition of frame structure and management information.
 3966 //
 3967 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3968 //                             |   (to get allocators register number
 3969 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3970 //  r   CALLER     |        |
 3971 //  o     |        +--------+      pad to even-align allocators stack-slot
 3972 //  w     V        |  pad0  |        numbers; owned by CALLER
 3973 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3974 //  h     ^        |   in   |  5
 3975 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3976 //  |     |        |        |  3
 3977 //  |     |        +--------+
 3978 //  V     |        | old out|      Empty on Intel, window on Sparc
 3979 //        |    old |preserve|      Must be even aligned.
 3980 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3981 //        |        |   in   |  3   area for Intel ret address
 3982 //     Owned by    |preserve|      Empty on Sparc.
 3983 //       SELF      +--------+
 3984 //        |        |  pad2  |  2   pad to align old SP
 3985 //        |        +--------+  1
 3986 //        |        | locks  |  0
 3987 //        |        +--------+----> OptoReg::stack0(), even aligned
 3988 //        |        |  pad1  | 11   pad to align new SP
 3989 //        |        +--------+
 3990 //        |        |        | 10
 3991 //        |        | spills |  9   spills
 3992 //        V        |        |  8   (pad0 slot for callee)
 3993 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3994 //        ^        |  out   |  7
 3995 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3996 //     Owned by    +--------+
 3997 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3998 //        |    new |preserve|      Must be even-aligned.
 3999 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4000 //        |        |        |
 4001 //
 4002 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4003 //         known from SELF's arguments and the Java calling convention.
 4004 //         Region 6-7 is determined per call site.
 4005 // Note 2: If the calling convention leaves holes in the incoming argument
 4006 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4007 //         are owned by the CALLEE.  Holes should not be nessecary in the
 4008 //         incoming area, as the Java calling convention is completely under
 4009 //         the control of the AD file.  Doubles can be sorted and packed to
 4010 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 4011 //         varargs C calling conventions.
 4012 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4013 //         even aligned with pad0 as needed.
 4014 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4015 //           (the latter is true on Intel but is it false on AArch64?)
 4016 //         region 6-11 is even aligned; it may be padded out more so that
 4017 //         the region from SP to FP meets the minimum stack alignment.
 4018 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4019 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4020 //         SP meets the minimum alignment.
 4021 
 4022 frame %{
 4023   // These three registers define part of the calling convention
 4024   // between compiled code and the interpreter.
 4025 
 4026   // Inline Cache Register or Method for I2C.
 4027   inline_cache_reg(R12);
 4028 
 4029   // Number of stack slots consumed by locking an object
 4030   sync_stack_slots(2);
 4031 
 4032   // Compiled code's Frame Pointer
 4033   frame_pointer(R31);
 4034 
 4035   // Interpreter stores its frame pointer in a register which is
 4036   // stored to the stack by I2CAdaptors.
 4037   // I2CAdaptors convert from interpreted java to compiled java.
 4038   interpreter_frame_pointer(R29);
 4039 
 4040   // Stack alignment requirement
 4041   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4042 
 4043   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4044   // for calls to C.  Supports the var-args backing area for register parms.
 4045   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4046 
 4047   // The after-PROLOG location of the return address.  Location of
 4048   // return address specifies a type (REG or STACK) and a number
 4049   // representing the register number (i.e. - use a register name) or
 4050   // stack slot.
 4051   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4052   // Otherwise, it is above the locks and verification slot and alignment word
 4053   // TODO this may well be correct but need to check why that - 2 is there
 4054   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4055   // which folds in the space used for monitors
 4056   return_addr(STACK - 2 +
 4057               align_up((Compile::current()->in_preserve_stack_slots() +
 4058                         Compile::current()->fixed_slots()),
 4059                        stack_alignment_in_slots()));
 4060 
 4061   // Location of compiled Java return values.  Same as C for now.
 4062   return_value
 4063   %{
 4064     // TODO do we allow ideal_reg == Op_RegN???
 4065     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4066            "only return normal values");
 4067 
 4068     static const int lo[Op_RegL + 1] = { // enum name
 4069       0,                                 // Op_Node
 4070       0,                                 // Op_Set
 4071       R0_num,                            // Op_RegN
 4072       R0_num,                            // Op_RegI
 4073       R0_num,                            // Op_RegP
 4074       V0_num,                            // Op_RegF
 4075       V0_num,                            // Op_RegD
 4076       R0_num                             // Op_RegL
 4077     };
 4078 
 4079     static const int hi[Op_RegL + 1] = { // enum name
 4080       0,                                 // Op_Node
 4081       0,                                 // Op_Set
 4082       OptoReg::Bad,                      // Op_RegN
 4083       OptoReg::Bad,                      // Op_RegI
 4084       R0_H_num,                          // Op_RegP
 4085       OptoReg::Bad,                      // Op_RegF
 4086       V0_H_num,                          // Op_RegD
 4087       R0_H_num                           // Op_RegL
 4088     };
 4089 
 4090     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4091   %}
 4092 %}
 4093 
 4094 //----------ATTRIBUTES---------------------------------------------------------
 4095 //----------Operand Attributes-------------------------------------------------
 4096 op_attrib op_cost(1);        // Required cost attribute
 4097 
 4098 //----------Instruction Attributes---------------------------------------------
 4099 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4100 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4101 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4102                                 // a non-matching short branch variant
 4103                                 // of some long branch?
 4104 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4105                                 // be a power of 2) specifies the
 4106                                 // alignment that some part of the
 4107                                 // instruction (not necessarily the
 4108                                 // start) requires.  If > 1, a
 4109                                 // compute_padding() function must be
 4110                                 // provided for the instruction
 4111 
 4112 //----------OPERANDS-----------------------------------------------------------
 4113 // Operand definitions must precede instruction definitions for correct parsing
 4114 // in the ADLC because operands constitute user defined types which are used in
 4115 // instruction definitions.
 4116 
 4117 //----------Simple Operands----------------------------------------------------
 4118 
 4119 // Integer operands 32 bit
 4120 // 32 bit immediate
 4121 operand immI()
 4122 %{
 4123   match(ConI);
 4124 
 4125   op_cost(0);
 4126   format %{ %}
 4127   interface(CONST_INTER);
 4128 %}
 4129 
 4130 // 32 bit zero
 4131 operand immI0()
 4132 %{
 4133   predicate(n->get_int() == 0);
 4134   match(ConI);
 4135 
 4136   op_cost(0);
 4137   format %{ %}
 4138   interface(CONST_INTER);
 4139 %}
 4140 
 4141 // 32 bit unit increment
 4142 operand immI_1()
 4143 %{
 4144   predicate(n->get_int() == 1);
 4145   match(ConI);
 4146 
 4147   op_cost(0);
 4148   format %{ %}
 4149   interface(CONST_INTER);
 4150 %}
 4151 
 4152 // 32 bit unit decrement
 4153 operand immI_M1()
 4154 %{
 4155   predicate(n->get_int() == -1);
 4156   match(ConI);
 4157 
 4158   op_cost(0);
 4159   format %{ %}
 4160   interface(CONST_INTER);
 4161 %}
 4162 
 4163 // Shift values for add/sub extension shift
 4164 operand immIExt()
 4165 %{
 4166   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4167   match(ConI);
 4168 
 4169   op_cost(0);
 4170   format %{ %}
 4171   interface(CONST_INTER);
 4172 %}
 4173 
 4174 operand immI_le_4()
 4175 %{
 4176   predicate(n->get_int() <= 4);
 4177   match(ConI);
 4178 
 4179   op_cost(0);
 4180   format %{ %}
 4181   interface(CONST_INTER);
 4182 %}
 4183 
 4184 operand immI_31()
 4185 %{
 4186   predicate(n->get_int() == 31);
 4187   match(ConI);
 4188 
 4189   op_cost(0);
 4190   format %{ %}
 4191   interface(CONST_INTER);
 4192 %}
 4193 
 4194 operand immI_2()
 4195 %{
 4196   predicate(n->get_int() == 2);
 4197   match(ConI);
 4198 
 4199   op_cost(0);
 4200   format %{ %}
 4201   interface(CONST_INTER);
 4202 %}
 4203 
 4204 operand immI_4()
 4205 %{
 4206   predicate(n->get_int() == 4);
 4207   match(ConI);
 4208 
 4209   op_cost(0);
 4210   format %{ %}
 4211   interface(CONST_INTER);
 4212 %}
 4213 
 4214 operand immI_8()
 4215 %{
 4216   predicate(n->get_int() == 8);
 4217   match(ConI);
 4218 
 4219   op_cost(0);
 4220   format %{ %}
 4221   interface(CONST_INTER);
 4222 %}
 4223 
 4224 operand immI_16()
 4225 %{
 4226   predicate(n->get_int() == 16);
 4227   match(ConI);
 4228 
 4229   op_cost(0);
 4230   format %{ %}
 4231   interface(CONST_INTER);
 4232 %}
 4233 
 4234 operand immI_24()
 4235 %{
 4236   predicate(n->get_int() == 24);
 4237   match(ConI);
 4238 
 4239   op_cost(0);
 4240   format %{ %}
 4241   interface(CONST_INTER);
 4242 %}
 4243 
 4244 operand immI_32()
 4245 %{
 4246   predicate(n->get_int() == 32);
 4247   match(ConI);
 4248 
 4249   op_cost(0);
 4250   format %{ %}
 4251   interface(CONST_INTER);
 4252 %}
 4253 
 4254 operand immI_48()
 4255 %{
 4256   predicate(n->get_int() == 48);
 4257   match(ConI);
 4258 
 4259   op_cost(0);
 4260   format %{ %}
 4261   interface(CONST_INTER);
 4262 %}
 4263 
 4264 operand immI_56()
 4265 %{
 4266   predicate(n->get_int() == 56);
 4267   match(ConI);
 4268 
 4269   op_cost(0);
 4270   format %{ %}
 4271   interface(CONST_INTER);
 4272 %}
 4273 
 4274 operand immI_63()
 4275 %{
 4276   predicate(n->get_int() == 63);
 4277   match(ConI);
 4278 
 4279   op_cost(0);
 4280   format %{ %}
 4281   interface(CONST_INTER);
 4282 %}
 4283 
 4284 operand immI_64()
 4285 %{
 4286   predicate(n->get_int() == 64);
 4287   match(ConI);
 4288 
 4289   op_cost(0);
 4290   format %{ %}
 4291   interface(CONST_INTER);
 4292 %}
 4293 
 4294 operand immI_255()
 4295 %{
 4296   predicate(n->get_int() == 255);
 4297   match(ConI);
 4298 
 4299   op_cost(0);
 4300   format %{ %}
 4301   interface(CONST_INTER);
 4302 %}
 4303 
 4304 operand immI_65535()
 4305 %{
 4306   predicate(n->get_int() == 65535);
 4307   match(ConI);
 4308 
 4309   op_cost(0);
 4310   format %{ %}
 4311   interface(CONST_INTER);
 4312 %}
 4313 
 4314 operand immI_positive()
 4315 %{
 4316   predicate(n->get_int() > 0);
 4317   match(ConI);
 4318 
 4319   op_cost(0);
 4320   format %{ %}
 4321   interface(CONST_INTER);
 4322 %}
 4323 
 4324 operand immL_255()
 4325 %{
 4326   predicate(n->get_long() == 255L);
 4327   match(ConL);
 4328 
 4329   op_cost(0);
 4330   format %{ %}
 4331   interface(CONST_INTER);
 4332 %}
 4333 
 4334 operand immL_65535()
 4335 %{
 4336   predicate(n->get_long() == 65535L);
 4337   match(ConL);
 4338 
 4339   op_cost(0);
 4340   format %{ %}
 4341   interface(CONST_INTER);
 4342 %}
 4343 
 4344 operand immL_4294967295()
 4345 %{
 4346   predicate(n->get_long() == 4294967295L);
 4347   match(ConL);
 4348 
 4349   op_cost(0);
 4350   format %{ %}
 4351   interface(CONST_INTER);
 4352 %}
 4353 
 4354 operand immL_bitmask()
 4355 %{
 4356   predicate((n->get_long() != 0)
 4357             && ((n->get_long() & 0xc000000000000000l) == 0)
 4358             && is_power_of_2(n->get_long() + 1));
 4359   match(ConL);
 4360 
 4361   op_cost(0);
 4362   format %{ %}
 4363   interface(CONST_INTER);
 4364 %}
 4365 
 4366 operand immI_bitmask()
 4367 %{
 4368   predicate((n->get_int() != 0)
 4369             && ((n->get_int() & 0xc0000000) == 0)
 4370             && is_power_of_2(n->get_int() + 1));
 4371   match(ConI);
 4372 
 4373   op_cost(0);
 4374   format %{ %}
 4375   interface(CONST_INTER);
 4376 %}
 4377 
 4378 operand immL_positive_bitmaskI()
 4379 %{
 4380   predicate((n->get_long() != 0)
 4381             && ((julong)n->get_long() < 0x80000000ULL)
 4382             && is_power_of_2(n->get_long() + 1));
 4383   match(ConL);
 4384 
 4385   op_cost(0);
 4386   format %{ %}
 4387   interface(CONST_INTER);
 4388 %}
 4389 
 4390 // Scale values for scaled offset addressing modes (up to long but not quad)
 4391 operand immIScale()
 4392 %{
 4393   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 // 26 bit signed offset -- for pc-relative branches
 4402 operand immI26()
 4403 %{
 4404   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4405   match(ConI);
 4406 
 4407   op_cost(0);
 4408   format %{ %}
 4409   interface(CONST_INTER);
 4410 %}
 4411 
 4412 // 19 bit signed offset -- for pc-relative loads
 4413 operand immI19()
 4414 %{
 4415   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4416   match(ConI);
 4417 
 4418   op_cost(0);
 4419   format %{ %}
 4420   interface(CONST_INTER);
 4421 %}
 4422 
 4423 // 12 bit unsigned offset -- for base plus immediate loads
 4424 operand immIU12()
 4425 %{
 4426   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4427   match(ConI);
 4428 
 4429   op_cost(0);
 4430   format %{ %}
 4431   interface(CONST_INTER);
 4432 %}
 4433 
 4434 operand immLU12()
 4435 %{
 4436   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4437   match(ConL);
 4438 
 4439   op_cost(0);
 4440   format %{ %}
 4441   interface(CONST_INTER);
 4442 %}
 4443 
 4444 // Offset for scaled or unscaled immediate loads and stores
 4445 operand immIOffset()
 4446 %{
 4447   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4448   match(ConI);
 4449 
 4450   op_cost(0);
 4451   format %{ %}
 4452   interface(CONST_INTER);
 4453 %}
 4454 
 4455 operand immIOffset1()
 4456 %{
 4457   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4458   match(ConI);
 4459 
 4460   op_cost(0);
 4461   format %{ %}
 4462   interface(CONST_INTER);
 4463 %}
 4464 
 4465 operand immIOffset2()
 4466 %{
 4467   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4468   match(ConI);
 4469 
 4470   op_cost(0);
 4471   format %{ %}
 4472   interface(CONST_INTER);
 4473 %}
 4474 
 4475 operand immIOffset4()
 4476 %{
 4477   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4478   match(ConI);
 4479 
 4480   op_cost(0);
 4481   format %{ %}
 4482   interface(CONST_INTER);
 4483 %}
 4484 
 4485 operand immIOffset8()
 4486 %{
 4487   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4488   match(ConI);
 4489 
 4490   op_cost(0);
 4491   format %{ %}
 4492   interface(CONST_INTER);
 4493 %}
 4494 
 4495 operand immIOffset16()
 4496 %{
 4497   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4498   match(ConI);
 4499 
 4500   op_cost(0);
 4501   format %{ %}
 4502   interface(CONST_INTER);
 4503 %}
 4504 
 4505 operand immLoffset()
 4506 %{
 4507   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4508   match(ConL);
 4509 
 4510   op_cost(0);
 4511   format %{ %}
 4512   interface(CONST_INTER);
 4513 %}
 4514 
 4515 operand immLoffset1()
 4516 %{
 4517   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4518   match(ConL);
 4519 
 4520   op_cost(0);
 4521   format %{ %}
 4522   interface(CONST_INTER);
 4523 %}
 4524 
 4525 operand immLoffset2()
 4526 %{
 4527   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4528   match(ConL);
 4529 
 4530   op_cost(0);
 4531   format %{ %}
 4532   interface(CONST_INTER);
 4533 %}
 4534 
 4535 operand immLoffset4()
 4536 %{
 4537   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4538   match(ConL);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 operand immLoffset8()
 4546 %{
 4547   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4548   match(ConL);
 4549 
 4550   op_cost(0);
 4551   format %{ %}
 4552   interface(CONST_INTER);
 4553 %}
 4554 
 4555 operand immLoffset16()
 4556 %{
 4557   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4558   match(ConL);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // 8 bit signed value.
 4566 operand immI8()
 4567 %{
 4568   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4569   match(ConI);
 4570 
 4571   op_cost(0);
 4572   format %{ %}
 4573   interface(CONST_INTER);
 4574 %}
 4575 
 4576 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4577 operand immI8_shift8()
 4578 %{
 4579   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4580             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4581   match(ConI);
 4582 
 4583   op_cost(0);
 4584   format %{ %}
 4585   interface(CONST_INTER);
 4586 %}
 4587 
 4588 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4589 operand immL8_shift8()
 4590 %{
 4591   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4592             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4593   match(ConL);
 4594 
 4595   op_cost(0);
 4596   format %{ %}
 4597   interface(CONST_INTER);
 4598 %}
 4599 
 4600 // 32 bit integer valid for add sub immediate
 4601 operand immIAddSub()
 4602 %{
 4603   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4604   match(ConI);
 4605   op_cost(0);
 4606   format %{ %}
 4607   interface(CONST_INTER);
 4608 %}
 4609 
 4610 // 32 bit unsigned integer valid for logical immediate
 4611 // TODO -- check this is right when e.g the mask is 0x80000000
 4612 operand immILog()
 4613 %{
 4614   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4615   match(ConI);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 // Integer operands 64 bit
 4623 // 64 bit immediate
 4624 operand immL()
 4625 %{
 4626   match(ConL);
 4627 
 4628   op_cost(0);
 4629   format %{ %}
 4630   interface(CONST_INTER);
 4631 %}
 4632 
 4633 // 64 bit zero
 4634 operand immL0()
 4635 %{
 4636   predicate(n->get_long() == 0);
 4637   match(ConL);
 4638 
 4639   op_cost(0);
 4640   format %{ %}
 4641   interface(CONST_INTER);
 4642 %}
 4643 
 4644 // 64 bit unit increment
 4645 operand immL_1()
 4646 %{
 4647   predicate(n->get_long() == 1);
 4648   match(ConL);
 4649 
 4650   op_cost(0);
 4651   format %{ %}
 4652   interface(CONST_INTER);
 4653 %}
 4654 
 4655 // 64 bit unit decrement
 4656 operand immL_M1()
 4657 %{
 4658   predicate(n->get_long() == -1);
 4659   match(ConL);
 4660 
 4661   op_cost(0);
 4662   format %{ %}
 4663   interface(CONST_INTER);
 4664 %}
 4665 
 4666 // 32 bit offset of pc in thread anchor
 4667 
 4668 operand immL_pc_off()
 4669 %{
 4670   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4671                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4672   match(ConL);
 4673 
 4674   op_cost(0);
 4675   format %{ %}
 4676   interface(CONST_INTER);
 4677 %}
 4678 
 4679 // 64 bit integer valid for add sub immediate
 4680 operand immLAddSub()
 4681 %{
 4682   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4683   match(ConL);
 4684   op_cost(0);
 4685   format %{ %}
 4686   interface(CONST_INTER);
 4687 %}
 4688 
 4689 // 64 bit integer valid for logical immediate
 4690 operand immLLog()
 4691 %{
 4692   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4693   match(ConL);
 4694   op_cost(0);
 4695   format %{ %}
 4696   interface(CONST_INTER);
 4697 %}
 4698 
 4699 // Long Immediate: low 32-bit mask
 4700 operand immL_32bits()
 4701 %{
 4702   predicate(n->get_long() == 0xFFFFFFFFL);
 4703   match(ConL);
 4704   op_cost(0);
 4705   format %{ %}
 4706   interface(CONST_INTER);
 4707 %}
 4708 
 4709 // Pointer operands
 4710 // Pointer Immediate
 4711 operand immP()
 4712 %{
 4713   match(ConP);
 4714 
 4715   op_cost(0);
 4716   format %{ %}
 4717   interface(CONST_INTER);
 4718 %}
 4719 
 4720 // NULL Pointer Immediate
 4721 operand immP0()
 4722 %{
 4723   predicate(n->get_ptr() == 0);
 4724   match(ConP);
 4725 
 4726   op_cost(0);
 4727   format %{ %}
 4728   interface(CONST_INTER);
 4729 %}
 4730 
 4731 // Pointer Immediate One
 4732 // this is used in object initialization (initial object header)
 4733 operand immP_1()
 4734 %{
 4735   predicate(n->get_ptr() == 1);
 4736   match(ConP);
 4737 
 4738   op_cost(0);
 4739   format %{ %}
 4740   interface(CONST_INTER);
 4741 %}
 4742 
 4743 // Card Table Byte Map Base
 4744 operand immByteMapBase()
 4745 %{
 4746   // Get base of card map
 4747   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4748             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4749   match(ConP);
 4750 
 4751   op_cost(0);
 4752   format %{ %}
 4753   interface(CONST_INTER);
 4754 %}
 4755 
 4756 // Pointer Immediate Minus One
 4757 // this is used when we want to write the current PC to the thread anchor
 4758 operand immP_M1()
 4759 %{
 4760   predicate(n->get_ptr() == -1);
 4761   match(ConP);
 4762 
 4763   op_cost(0);
 4764   format %{ %}
 4765   interface(CONST_INTER);
 4766 %}
 4767 
 4768 // Pointer Immediate Minus Two
 4769 // this is used when we want to write the current PC to the thread anchor
 4770 operand immP_M2()
 4771 %{
 4772   predicate(n->get_ptr() == -2);
 4773   match(ConP);
 4774 
 4775   op_cost(0);
 4776   format %{ %}
 4777   interface(CONST_INTER);
 4778 %}
 4779 
 4780 // Float and Double operands
 4781 // Double Immediate
 4782 operand immD()
 4783 %{
 4784   match(ConD);
 4785   op_cost(0);
 4786   format %{ %}
 4787   interface(CONST_INTER);
 4788 %}
 4789 
 4790 // Double Immediate: +0.0d
 4791 operand immD0()
 4792 %{
 4793   predicate(jlong_cast(n->getd()) == 0);
 4794   match(ConD);
 4795 
 4796   op_cost(0);
 4797   format %{ %}
 4798   interface(CONST_INTER);
 4799 %}
 4800 
 4801 // constant 'double +0.0'.
 4802 operand immDPacked()
 4803 %{
 4804   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4805   match(ConD);
 4806   op_cost(0);
 4807   format %{ %}
 4808   interface(CONST_INTER);
 4809 %}
 4810 
 4811 // Float Immediate
 4812 operand immF()
 4813 %{
 4814   match(ConF);
 4815   op_cost(0);
 4816   format %{ %}
 4817   interface(CONST_INTER);
 4818 %}
 4819 
 4820 // Float Immediate: +0.0f.
 4821 operand immF0()
 4822 %{
 4823   predicate(jint_cast(n->getf()) == 0);
 4824   match(ConF);
 4825 
 4826   op_cost(0);
 4827   format %{ %}
 4828   interface(CONST_INTER);
 4829 %}
 4830 
 4831 //
 4832 operand immFPacked()
 4833 %{
 4834   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4835   match(ConF);
 4836   op_cost(0);
 4837   format %{ %}
 4838   interface(CONST_INTER);
 4839 %}
 4840 
 4841 // Narrow pointer operands
 4842 // Narrow Pointer Immediate
 4843 operand immN()
 4844 %{
 4845   match(ConN);
 4846 
 4847   op_cost(0);
 4848   format %{ %}
 4849   interface(CONST_INTER);
 4850 %}
 4851 
 4852 // Narrow NULL Pointer Immediate
 4853 operand immN0()
 4854 %{
 4855   predicate(n->get_narrowcon() == 0);
 4856   match(ConN);
 4857 
 4858   op_cost(0);
 4859   format %{ %}
 4860   interface(CONST_INTER);
 4861 %}
 4862 
 4863 operand immNKlass()
 4864 %{
 4865   match(ConNKlass);
 4866 
 4867   op_cost(0);
 4868   format %{ %}
 4869   interface(CONST_INTER);
 4870 %}
 4871 
 4872 // Integer 32 bit Register Operands
 4873 // Integer 32 bitRegister (excludes SP)
 4874 operand iRegI()
 4875 %{
 4876   constraint(ALLOC_IN_RC(any_reg32));
 4877   match(RegI);
 4878   match(iRegINoSp);
 4879   op_cost(0);
 4880   format %{ %}
 4881   interface(REG_INTER);
 4882 %}
 4883 
 4884 // Integer 32 bit Register not Special
 4885 operand iRegINoSp()
 4886 %{
 4887   constraint(ALLOC_IN_RC(no_special_reg32));
 4888   match(RegI);
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(REG_INTER);
 4892 %}
 4893 
 4894 // Integer 64 bit Register Operands
 4895 // Integer 64 bit Register (includes SP)
 4896 operand iRegL()
 4897 %{
 4898   constraint(ALLOC_IN_RC(any_reg));
 4899   match(RegL);
 4900   match(iRegLNoSp);
 4901   op_cost(0);
 4902   format %{ %}
 4903   interface(REG_INTER);
 4904 %}
 4905 
 4906 // Integer 64 bit Register not Special
 4907 operand iRegLNoSp()
 4908 %{
 4909   constraint(ALLOC_IN_RC(no_special_reg));
 4910   match(RegL);
 4911   match(iRegL_R0);
 4912   format %{ %}
 4913   interface(REG_INTER);
 4914 %}
 4915 
 4916 // Pointer Register Operands
 4917 // Pointer Register
 4918 operand iRegP()
 4919 %{
 4920   constraint(ALLOC_IN_RC(ptr_reg));
 4921   match(RegP);
 4922   match(iRegPNoSp);
 4923   match(iRegP_R0);
 4924   //match(iRegP_R2);
 4925   //match(iRegP_R4);
 4926   match(iRegP_R5);
 4927   match(thread_RegP);
 4928   op_cost(0);
 4929   format %{ %}
 4930   interface(REG_INTER);
 4931 %}
 4932 
 4933 // Pointer 64 bit Register not Special
 4934 operand iRegPNoSp()
 4935 %{
 4936   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4937   match(RegP);
 4938   // match(iRegP);
 4939   // match(iRegP_R0);
 4940   // match(iRegP_R2);
 4941   // match(iRegP_R4);
 4942   // match(iRegP_R5);
 4943   // match(thread_RegP);
 4944   op_cost(0);
 4945   format %{ %}
 4946   interface(REG_INTER);
 4947 %}
 4948 
 4949 // Pointer 64 bit Register R0 only
 4950 operand iRegP_R0()
 4951 %{
 4952   constraint(ALLOC_IN_RC(r0_reg));
 4953   match(RegP);
 4954   // match(iRegP);
 4955   match(iRegPNoSp);
 4956   op_cost(0);
 4957   format %{ %}
 4958   interface(REG_INTER);
 4959 %}
 4960 
 4961 // Pointer 64 bit Register R1 only
 4962 operand iRegP_R1()
 4963 %{
 4964   constraint(ALLOC_IN_RC(r1_reg));
 4965   match(RegP);
 4966   // match(iRegP);
 4967   match(iRegPNoSp);
 4968   op_cost(0);
 4969   format %{ %}
 4970   interface(REG_INTER);
 4971 %}
 4972 
 4973 // Pointer 64 bit Register R2 only
 4974 operand iRegP_R2()
 4975 %{
 4976   constraint(ALLOC_IN_RC(r2_reg));
 4977   match(RegP);
 4978   // match(iRegP);
 4979   match(iRegPNoSp);
 4980   op_cost(0);
 4981   format %{ %}
 4982   interface(REG_INTER);
 4983 %}
 4984 
 4985 // Pointer 64 bit Register R3 only
 4986 operand iRegP_R3()
 4987 %{
 4988   constraint(ALLOC_IN_RC(r3_reg));
 4989   match(RegP);
 4990   // match(iRegP);
 4991   match(iRegPNoSp);
 4992   op_cost(0);
 4993   format %{ %}
 4994   interface(REG_INTER);
 4995 %}
 4996 
 4997 // Pointer 64 bit Register R4 only
 4998 operand iRegP_R4()
 4999 %{
 5000   constraint(ALLOC_IN_RC(r4_reg));
 5001   match(RegP);
 5002   // match(iRegP);
 5003   match(iRegPNoSp);
 5004   op_cost(0);
 5005   format %{ %}
 5006   interface(REG_INTER);
 5007 %}
 5008 
 5009 // Pointer 64 bit Register R5 only
 5010 operand iRegP_R5()
 5011 %{
 5012   constraint(ALLOC_IN_RC(r5_reg));
 5013   match(RegP);
 5014   // match(iRegP);
 5015   match(iRegPNoSp);
 5016   op_cost(0);
 5017   format %{ %}
 5018   interface(REG_INTER);
 5019 %}
 5020 
 5021 // Pointer 64 bit Register R10 only
 5022 operand iRegP_R10()
 5023 %{
 5024   constraint(ALLOC_IN_RC(r10_reg));
 5025   match(RegP);
 5026   // match(iRegP);
 5027   match(iRegPNoSp);
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(REG_INTER);
 5031 %}
 5032 
 5033 // Long 64 bit Register R0 only
 5034 operand iRegL_R0()
 5035 %{
 5036   constraint(ALLOC_IN_RC(r0_reg));
 5037   match(RegL);
 5038   match(iRegLNoSp);
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(REG_INTER);
 5042 %}
 5043 
 5044 // Long 64 bit Register R2 only
 5045 operand iRegL_R2()
 5046 %{
 5047   constraint(ALLOC_IN_RC(r2_reg));
 5048   match(RegL);
 5049   match(iRegLNoSp);
 5050   op_cost(0);
 5051   format %{ %}
 5052   interface(REG_INTER);
 5053 %}
 5054 
 5055 // Long 64 bit Register R3 only
 5056 operand iRegL_R3()
 5057 %{
 5058   constraint(ALLOC_IN_RC(r3_reg));
 5059   match(RegL);
 5060   match(iRegLNoSp);
 5061   op_cost(0);
 5062   format %{ %}
 5063   interface(REG_INTER);
 5064 %}
 5065 
 5066 // Long 64 bit Register R11 only
 5067 operand iRegL_R11()
 5068 %{
 5069   constraint(ALLOC_IN_RC(r11_reg));
 5070   match(RegL);
 5071   match(iRegLNoSp);
 5072   op_cost(0);
 5073   format %{ %}
 5074   interface(REG_INTER);
 5075 %}
 5076 
 5077 // Pointer 64 bit Register FP only
 5078 operand iRegP_FP()
 5079 %{
 5080   constraint(ALLOC_IN_RC(fp_reg));
 5081   match(RegP);
 5082   // match(iRegP);
 5083   op_cost(0);
 5084   format %{ %}
 5085   interface(REG_INTER);
 5086 %}
 5087 
 5088 // Register R0 only
 5089 operand iRegI_R0()
 5090 %{
 5091   constraint(ALLOC_IN_RC(int_r0_reg));
 5092   match(RegI);
 5093   match(iRegINoSp);
 5094   op_cost(0);
 5095   format %{ %}
 5096   interface(REG_INTER);
 5097 %}
 5098 
 5099 // Register R2 only
 5100 operand iRegI_R2()
 5101 %{
 5102   constraint(ALLOC_IN_RC(int_r2_reg));
 5103   match(RegI);
 5104   match(iRegINoSp);
 5105   op_cost(0);
 5106   format %{ %}
 5107   interface(REG_INTER);
 5108 %}
 5109 
 5110 // Register R3 only
 5111 operand iRegI_R3()
 5112 %{
 5113   constraint(ALLOC_IN_RC(int_r3_reg));
 5114   match(RegI);
 5115   match(iRegINoSp);
 5116   op_cost(0);
 5117   format %{ %}
 5118   interface(REG_INTER);
 5119 %}
 5120 
 5121 
 5122 // Register R4 only
 5123 operand iRegI_R4()
 5124 %{
 5125   constraint(ALLOC_IN_RC(int_r4_reg));
 5126   match(RegI);
 5127   match(iRegINoSp);
 5128   op_cost(0);
 5129   format %{ %}
 5130   interface(REG_INTER);
 5131 %}
 5132 
 5133 
 5134 // Pointer Register Operands
 5135 // Narrow Pointer Register
 5136 operand iRegN()
 5137 %{
 5138   constraint(ALLOC_IN_RC(any_reg32));
 5139   match(RegN);
 5140   match(iRegNNoSp);
 5141   op_cost(0);
 5142   format %{ %}
 5143   interface(REG_INTER);
 5144 %}
 5145 
 5146 operand iRegN_R0()
 5147 %{
 5148   constraint(ALLOC_IN_RC(r0_reg));
 5149   match(iRegN);
 5150   op_cost(0);
 5151   format %{ %}
 5152   interface(REG_INTER);
 5153 %}
 5154 
 5155 operand iRegN_R2()
 5156 %{
 5157   constraint(ALLOC_IN_RC(r2_reg));
 5158   match(iRegN);
 5159   op_cost(0);
 5160   format %{ %}
 5161   interface(REG_INTER);
 5162 %}
 5163 
 5164 operand iRegN_R3()
 5165 %{
 5166   constraint(ALLOC_IN_RC(r3_reg));
 5167   match(iRegN);
 5168   op_cost(0);
 5169   format %{ %}
 5170   interface(REG_INTER);
 5171 %}
 5172 
 5173 // Integer 64 bit Register not Special
 5174 operand iRegNNoSp()
 5175 %{
 5176   constraint(ALLOC_IN_RC(no_special_reg32));
 5177   match(RegN);
 5178   op_cost(0);
 5179   format %{ %}
 5180   interface(REG_INTER);
 5181 %}
 5182 
 5183 // heap base register -- used for encoding immN0
 5184 
 5185 operand iRegIHeapbase()
 5186 %{
 5187   constraint(ALLOC_IN_RC(heapbase_reg));
 5188   match(RegI);
 5189   op_cost(0);
 5190   format %{ %}
 5191   interface(REG_INTER);
 5192 %}
 5193 
 5194 // Float Register
 5195 // Float register operands
 5196 operand vRegF()
 5197 %{
 5198   constraint(ALLOC_IN_RC(float_reg));
 5199   match(RegF);
 5200 
 5201   op_cost(0);
 5202   format %{ %}
 5203   interface(REG_INTER);
 5204 %}
 5205 
 5206 // Double Register
 5207 // Double register operands
 5208 operand vRegD()
 5209 %{
 5210   constraint(ALLOC_IN_RC(double_reg));
 5211   match(RegD);
 5212 
 5213   op_cost(0);
 5214   format %{ %}
 5215   interface(REG_INTER);
 5216 %}
 5217 
 5218 // Generic vector class. This will be used for
 5219 // all vector operands, including NEON and SVE,
 5220 // but currently only used for SVE VecA.
 5221 operand vReg()
 5222 %{
 5223   constraint(ALLOC_IN_RC(vectora_reg));
 5224   match(VecA);
 5225   op_cost(0);
 5226   format %{ %}
 5227   interface(REG_INTER);
 5228 %}
 5229 
 5230 operand vecD()
 5231 %{
 5232   constraint(ALLOC_IN_RC(vectord_reg));
 5233   match(VecD);
 5234 
 5235   op_cost(0);
 5236   format %{ %}
 5237   interface(REG_INTER);
 5238 %}
 5239 
 5240 operand vecX()
 5241 %{
 5242   constraint(ALLOC_IN_RC(vectorx_reg));
 5243   match(VecX);
 5244 
 5245   op_cost(0);
 5246   format %{ %}
 5247   interface(REG_INTER);
 5248 %}
 5249 
 5250 operand vRegD_V0()
 5251 %{
 5252   constraint(ALLOC_IN_RC(v0_reg));
 5253   match(RegD);
 5254   op_cost(0);
 5255   format %{ %}
 5256   interface(REG_INTER);
 5257 %}
 5258 
 5259 operand vRegD_V1()
 5260 %{
 5261   constraint(ALLOC_IN_RC(v1_reg));
 5262   match(RegD);
 5263   op_cost(0);
 5264   format %{ %}
 5265   interface(REG_INTER);
 5266 %}
 5267 
 5268 operand vRegD_V2()
 5269 %{
 5270   constraint(ALLOC_IN_RC(v2_reg));
 5271   match(RegD);
 5272   op_cost(0);
 5273   format %{ %}
 5274   interface(REG_INTER);
 5275 %}
 5276 
 5277 operand vRegD_V3()
 5278 %{
 5279   constraint(ALLOC_IN_RC(v3_reg));
 5280   match(RegD);
 5281   op_cost(0);
 5282   format %{ %}
 5283   interface(REG_INTER);
 5284 %}
 5285 
 5286 operand vRegD_V4()
 5287 %{
 5288   constraint(ALLOC_IN_RC(v4_reg));
 5289   match(RegD);
 5290   op_cost(0);
 5291   format %{ %}
 5292   interface(REG_INTER);
 5293 %}
 5294 
 5295 operand vRegD_V5()
 5296 %{
 5297   constraint(ALLOC_IN_RC(v5_reg));
 5298   match(RegD);
 5299   op_cost(0);
 5300   format %{ %}
 5301   interface(REG_INTER);
 5302 %}
 5303 
 5304 operand vRegD_V6()
 5305 %{
 5306   constraint(ALLOC_IN_RC(v6_reg));
 5307   match(RegD);
 5308   op_cost(0);
 5309   format %{ %}
 5310   interface(REG_INTER);
 5311 %}
 5312 
 5313 operand vRegD_V7()
 5314 %{
 5315   constraint(ALLOC_IN_RC(v7_reg));
 5316   match(RegD);
 5317   op_cost(0);
 5318   format %{ %}
 5319   interface(REG_INTER);
 5320 %}
 5321 
 5322 operand vRegD_V8()
 5323 %{
 5324   constraint(ALLOC_IN_RC(v8_reg));
 5325   match(RegD);
 5326   op_cost(0);
 5327   format %{ %}
 5328   interface(REG_INTER);
 5329 %}
 5330 
 5331 operand vRegD_V9()
 5332 %{
 5333   constraint(ALLOC_IN_RC(v9_reg));
 5334   match(RegD);
 5335   op_cost(0);
 5336   format %{ %}
 5337   interface(REG_INTER);
 5338 %}
 5339 
 5340 operand vRegD_V10()
 5341 %{
 5342   constraint(ALLOC_IN_RC(v10_reg));
 5343   match(RegD);
 5344   op_cost(0);
 5345   format %{ %}
 5346   interface(REG_INTER);
 5347 %}
 5348 
 5349 operand vRegD_V11()
 5350 %{
 5351   constraint(ALLOC_IN_RC(v11_reg));
 5352   match(RegD);
 5353   op_cost(0);
 5354   format %{ %}
 5355   interface(REG_INTER);
 5356 %}
 5357 
 5358 operand vRegD_V12()
 5359 %{
 5360   constraint(ALLOC_IN_RC(v12_reg));
 5361   match(RegD);
 5362   op_cost(0);
 5363   format %{ %}
 5364   interface(REG_INTER);
 5365 %}
 5366 
 5367 operand vRegD_V13()
 5368 %{
 5369   constraint(ALLOC_IN_RC(v13_reg));
 5370   match(RegD);
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vRegD_V14()
 5377 %{
 5378   constraint(ALLOC_IN_RC(v14_reg));
 5379   match(RegD);
 5380   op_cost(0);
 5381   format %{ %}
 5382   interface(REG_INTER);
 5383 %}
 5384 
 5385 operand vRegD_V15()
 5386 %{
 5387   constraint(ALLOC_IN_RC(v15_reg));
 5388   match(RegD);
 5389   op_cost(0);
 5390   format %{ %}
 5391   interface(REG_INTER);
 5392 %}
 5393 
 5394 operand vRegD_V16()
 5395 %{
 5396   constraint(ALLOC_IN_RC(v16_reg));
 5397   match(RegD);
 5398   op_cost(0);
 5399   format %{ %}
 5400   interface(REG_INTER);
 5401 %}
 5402 
 5403 operand vRegD_V17()
 5404 %{
 5405   constraint(ALLOC_IN_RC(v17_reg));
 5406   match(RegD);
 5407   op_cost(0);
 5408   format %{ %}
 5409   interface(REG_INTER);
 5410 %}
 5411 
 5412 operand vRegD_V18()
 5413 %{
 5414   constraint(ALLOC_IN_RC(v18_reg));
 5415   match(RegD);
 5416   op_cost(0);
 5417   format %{ %}
 5418   interface(REG_INTER);
 5419 %}
 5420 
 5421 operand vRegD_V19()
 5422 %{
 5423   constraint(ALLOC_IN_RC(v19_reg));
 5424   match(RegD);
 5425   op_cost(0);
 5426   format %{ %}
 5427   interface(REG_INTER);
 5428 %}
 5429 
 5430 operand vRegD_V20()
 5431 %{
 5432   constraint(ALLOC_IN_RC(v20_reg));
 5433   match(RegD);
 5434   op_cost(0);
 5435   format %{ %}
 5436   interface(REG_INTER);
 5437 %}
 5438 
 5439 operand vRegD_V21()
 5440 %{
 5441   constraint(ALLOC_IN_RC(v21_reg));
 5442   match(RegD);
 5443   op_cost(0);
 5444   format %{ %}
 5445   interface(REG_INTER);
 5446 %}
 5447 
 5448 operand vRegD_V22()
 5449 %{
 5450   constraint(ALLOC_IN_RC(v22_reg));
 5451   match(RegD);
 5452   op_cost(0);
 5453   format %{ %}
 5454   interface(REG_INTER);
 5455 %}
 5456 
 5457 operand vRegD_V23()
 5458 %{
 5459   constraint(ALLOC_IN_RC(v23_reg));
 5460   match(RegD);
 5461   op_cost(0);
 5462   format %{ %}
 5463   interface(REG_INTER);
 5464 %}
 5465 
 5466 operand vRegD_V24()
 5467 %{
 5468   constraint(ALLOC_IN_RC(v24_reg));
 5469   match(RegD);
 5470   op_cost(0);
 5471   format %{ %}
 5472   interface(REG_INTER);
 5473 %}
 5474 
 5475 operand vRegD_V25()
 5476 %{
 5477   constraint(ALLOC_IN_RC(v25_reg));
 5478   match(RegD);
 5479   op_cost(0);
 5480   format %{ %}
 5481   interface(REG_INTER);
 5482 %}
 5483 
 5484 operand vRegD_V26()
 5485 %{
 5486   constraint(ALLOC_IN_RC(v26_reg));
 5487   match(RegD);
 5488   op_cost(0);
 5489   format %{ %}
 5490   interface(REG_INTER);
 5491 %}
 5492 
 5493 operand vRegD_V27()
 5494 %{
 5495   constraint(ALLOC_IN_RC(v27_reg));
 5496   match(RegD);
 5497   op_cost(0);
 5498   format %{ %}
 5499   interface(REG_INTER);
 5500 %}
 5501 
 5502 operand vRegD_V28()
 5503 %{
 5504   constraint(ALLOC_IN_RC(v28_reg));
 5505   match(RegD);
 5506   op_cost(0);
 5507   format %{ %}
 5508   interface(REG_INTER);
 5509 %}
 5510 
 5511 operand vRegD_V29()
 5512 %{
 5513   constraint(ALLOC_IN_RC(v29_reg));
 5514   match(RegD);
 5515   op_cost(0);
 5516   format %{ %}
 5517   interface(REG_INTER);
 5518 %}
 5519 
 5520 operand vRegD_V30()
 5521 %{
 5522   constraint(ALLOC_IN_RC(v30_reg));
 5523   match(RegD);
 5524   op_cost(0);
 5525   format %{ %}
 5526   interface(REG_INTER);
 5527 %}
 5528 
 5529 operand vRegD_V31()
 5530 %{
 5531   constraint(ALLOC_IN_RC(v31_reg));
 5532   match(RegD);
 5533   op_cost(0);
 5534   format %{ %}
 5535   interface(REG_INTER);
 5536 %}
 5537 
 5538 operand pRegGov()
 5539 %{
 5540   constraint(ALLOC_IN_RC(gov_pr));
 5541   match(RegVectMask);
 5542   op_cost(0);
 5543   format %{ %}
 5544   interface(REG_INTER);
 5545 %}
 5546 
 5547 // Flags register, used as output of signed compare instructions
 5548 
 5549 // note that on AArch64 we also use this register as the output for
 5550 // for floating point compare instructions (CmpF CmpD). this ensures
 5551 // that ordered inequality tests use GT, GE, LT or LE none of which
 5552 // pass through cases where the result is unordered i.e. one or both
 5553 // inputs to the compare is a NaN. this means that the ideal code can
 5554 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5555 // (where the comparison should always fail). EQ and NE tests are
 5556 // always generated in ideal code so that unordered folds into the NE
 5557 // case, matching the behaviour of AArch64 NE.
 5558 //
 5559 // This differs from x86 where the outputs of FP compares use a
 5560 // special FP flags registers and where compares based on this
 5561 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5562 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5563 // to explicitly handle the unordered case in branches. x86 also has
 5564 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5565 
 5566 operand rFlagsReg()
 5567 %{
 5568   constraint(ALLOC_IN_RC(int_flags));
 5569   match(RegFlags);
 5570 
 5571   op_cost(0);
 5572   format %{ "RFLAGS" %}
 5573   interface(REG_INTER);
 5574 %}
 5575 
 5576 // Flags register, used as output of unsigned compare instructions
 5577 operand rFlagsRegU()
 5578 %{
 5579   constraint(ALLOC_IN_RC(int_flags));
 5580   match(RegFlags);
 5581 
 5582   op_cost(0);
 5583   format %{ "RFLAGSU" %}
 5584   interface(REG_INTER);
 5585 %}
 5586 
 5587 // Special Registers
 5588 
 5589 // Method Register
 5590 operand inline_cache_RegP(iRegP reg)
 5591 %{
 5592   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5593   match(reg);
 5594   match(iRegPNoSp);
 5595   op_cost(0);
 5596   format %{ %}
 5597   interface(REG_INTER);
 5598 %}
 5599 
 5600 // Thread Register
 5601 operand thread_RegP(iRegP reg)
 5602 %{
 5603   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5604   match(reg);
 5605   op_cost(0);
 5606   format %{ %}
 5607   interface(REG_INTER);
 5608 %}
 5609 
 5610 operand lr_RegP(iRegP reg)
 5611 %{
 5612   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5613   match(reg);
 5614   op_cost(0);
 5615   format %{ %}
 5616   interface(REG_INTER);
 5617 %}
 5618 
 5619 //----------Memory Operands----------------------------------------------------
 5620 
 5621 operand indirect(iRegP reg)
 5622 %{
 5623   constraint(ALLOC_IN_RC(ptr_reg));
 5624   match(reg);
 5625   op_cost(0);
 5626   format %{ "[$reg]" %}
 5627   interface(MEMORY_INTER) %{
 5628     base($reg);
 5629     index(0xffffffff);
 5630     scale(0x0);
 5631     disp(0x0);
 5632   %}
 5633 %}
 5634 
 5635 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5636 %{
 5637   constraint(ALLOC_IN_RC(ptr_reg));
 5638   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5639   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5640   op_cost(0);
 5641   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5642   interface(MEMORY_INTER) %{
 5643     base($reg);
 5644     index($ireg);
 5645     scale($scale);
 5646     disp(0x0);
 5647   %}
 5648 %}
 5649 
 5650 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5651 %{
 5652   constraint(ALLOC_IN_RC(ptr_reg));
 5653   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5654   match(AddP reg (LShiftL lreg scale));
 5655   op_cost(0);
 5656   format %{ "$reg, $lreg lsl($scale)" %}
 5657   interface(MEMORY_INTER) %{
 5658     base($reg);
 5659     index($lreg);
 5660     scale($scale);
 5661     disp(0x0);
 5662   %}
 5663 %}
 5664 
 5665 operand indIndexI2L(iRegP reg, iRegI ireg)
 5666 %{
 5667   constraint(ALLOC_IN_RC(ptr_reg));
 5668   match(AddP reg (ConvI2L ireg));
 5669   op_cost(0);
 5670   format %{ "$reg, $ireg, 0, I2L" %}
 5671   interface(MEMORY_INTER) %{
 5672     base($reg);
 5673     index($ireg);
 5674     scale(0x0);
 5675     disp(0x0);
 5676   %}
 5677 %}
 5678 
 5679 operand indIndex(iRegP reg, iRegL lreg)
 5680 %{
 5681   constraint(ALLOC_IN_RC(ptr_reg));
 5682   match(AddP reg lreg);
 5683   op_cost(0);
 5684   format %{ "$reg, $lreg" %}
 5685   interface(MEMORY_INTER) %{
 5686     base($reg);
 5687     index($lreg);
 5688     scale(0x0);
 5689     disp(0x0);
 5690   %}
 5691 %}
 5692 
 5693 operand indOffI(iRegP reg, immIOffset off)
 5694 %{
 5695   constraint(ALLOC_IN_RC(ptr_reg));
 5696   match(AddP reg off);
 5697   op_cost(0);
 5698   format %{ "[$reg, $off]" %}
 5699   interface(MEMORY_INTER) %{
 5700     base($reg);
 5701     index(0xffffffff);
 5702     scale(0x0);
 5703     disp($off);
 5704   %}
 5705 %}
 5706 
 5707 operand indOffI1(iRegP reg, immIOffset1 off)
 5708 %{
 5709   constraint(ALLOC_IN_RC(ptr_reg));
 5710   match(AddP reg off);
 5711   op_cost(0);
 5712   format %{ "[$reg, $off]" %}
 5713   interface(MEMORY_INTER) %{
 5714     base($reg);
 5715     index(0xffffffff);
 5716     scale(0x0);
 5717     disp($off);
 5718   %}
 5719 %}
 5720 
 5721 operand indOffI2(iRegP reg, immIOffset2 off)
 5722 %{
 5723   constraint(ALLOC_IN_RC(ptr_reg));
 5724   match(AddP reg off);
 5725   op_cost(0);
 5726   format %{ "[$reg, $off]" %}
 5727   interface(MEMORY_INTER) %{
 5728     base($reg);
 5729     index(0xffffffff);
 5730     scale(0x0);
 5731     disp($off);
 5732   %}
 5733 %}
 5734 
 5735 operand indOffI4(iRegP reg, immIOffset4 off)
 5736 %{
 5737   constraint(ALLOC_IN_RC(ptr_reg));
 5738   match(AddP reg off);
 5739   op_cost(0);
 5740   format %{ "[$reg, $off]" %}
 5741   interface(MEMORY_INTER) %{
 5742     base($reg);
 5743     index(0xffffffff);
 5744     scale(0x0);
 5745     disp($off);
 5746   %}
 5747 %}
 5748 
 5749 operand indOffI8(iRegP reg, immIOffset8 off)
 5750 %{
 5751   constraint(ALLOC_IN_RC(ptr_reg));
 5752   match(AddP reg off);
 5753   op_cost(0);
 5754   format %{ "[$reg, $off]" %}
 5755   interface(MEMORY_INTER) %{
 5756     base($reg);
 5757     index(0xffffffff);
 5758     scale(0x0);
 5759     disp($off);
 5760   %}
 5761 %}
 5762 
 5763 operand indOffI16(iRegP reg, immIOffset16 off)
 5764 %{
 5765   constraint(ALLOC_IN_RC(ptr_reg));
 5766   match(AddP reg off);
 5767   op_cost(0);
 5768   format %{ "[$reg, $off]" %}
 5769   interface(MEMORY_INTER) %{
 5770     base($reg);
 5771     index(0xffffffff);
 5772     scale(0x0);
 5773     disp($off);
 5774   %}
 5775 %}
 5776 
 5777 operand indOffL(iRegP reg, immLoffset off)
 5778 %{
 5779   constraint(ALLOC_IN_RC(ptr_reg));
 5780   match(AddP reg off);
 5781   op_cost(0);
 5782   format %{ "[$reg, $off]" %}
 5783   interface(MEMORY_INTER) %{
 5784     base($reg);
 5785     index(0xffffffff);
 5786     scale(0x0);
 5787     disp($off);
 5788   %}
 5789 %}
 5790 
 5791 operand indOffL1(iRegP reg, immLoffset1 off)
 5792 %{
 5793   constraint(ALLOC_IN_RC(ptr_reg));
 5794   match(AddP reg off);
 5795   op_cost(0);
 5796   format %{ "[$reg, $off]" %}
 5797   interface(MEMORY_INTER) %{
 5798     base($reg);
 5799     index(0xffffffff);
 5800     scale(0x0);
 5801     disp($off);
 5802   %}
 5803 %}
 5804 
 5805 operand indOffL2(iRegP reg, immLoffset2 off)
 5806 %{
 5807   constraint(ALLOC_IN_RC(ptr_reg));
 5808   match(AddP reg off);
 5809   op_cost(0);
 5810   format %{ "[$reg, $off]" %}
 5811   interface(MEMORY_INTER) %{
 5812     base($reg);
 5813     index(0xffffffff);
 5814     scale(0x0);
 5815     disp($off);
 5816   %}
 5817 %}
 5818 
 5819 operand indOffL4(iRegP reg, immLoffset4 off)
 5820 %{
 5821   constraint(ALLOC_IN_RC(ptr_reg));
 5822   match(AddP reg off);
 5823   op_cost(0);
 5824   format %{ "[$reg, $off]" %}
 5825   interface(MEMORY_INTER) %{
 5826     base($reg);
 5827     index(0xffffffff);
 5828     scale(0x0);
 5829     disp($off);
 5830   %}
 5831 %}
 5832 
 5833 operand indOffL8(iRegP reg, immLoffset8 off)
 5834 %{
 5835   constraint(ALLOC_IN_RC(ptr_reg));
 5836   match(AddP reg off);
 5837   op_cost(0);
 5838   format %{ "[$reg, $off]" %}
 5839   interface(MEMORY_INTER) %{
 5840     base($reg);
 5841     index(0xffffffff);
 5842     scale(0x0);
 5843     disp($off);
 5844   %}
 5845 %}
 5846 
 5847 operand indOffL16(iRegP reg, immLoffset16 off)
 5848 %{
 5849   constraint(ALLOC_IN_RC(ptr_reg));
 5850   match(AddP reg off);
 5851   op_cost(0);
 5852   format %{ "[$reg, $off]" %}
 5853   interface(MEMORY_INTER) %{
 5854     base($reg);
 5855     index(0xffffffff);
 5856     scale(0x0);
 5857     disp($off);
 5858   %}
 5859 %}
 5860 
 5861 operand indirectN(iRegN reg)
 5862 %{
 5863   predicate(CompressedOops::shift() == 0);
 5864   constraint(ALLOC_IN_RC(ptr_reg));
 5865   match(DecodeN reg);
 5866   op_cost(0);
 5867   format %{ "[$reg]\t# narrow" %}
 5868   interface(MEMORY_INTER) %{
 5869     base($reg);
 5870     index(0xffffffff);
 5871     scale(0x0);
 5872     disp(0x0);
 5873   %}
 5874 %}
 5875 
 5876 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5877 %{
 5878   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5879   constraint(ALLOC_IN_RC(ptr_reg));
 5880   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5881   op_cost(0);
 5882   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5883   interface(MEMORY_INTER) %{
 5884     base($reg);
 5885     index($ireg);
 5886     scale($scale);
 5887     disp(0x0);
 5888   %}
 5889 %}
 5890 
 5891 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5892 %{
 5893   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5894   constraint(ALLOC_IN_RC(ptr_reg));
 5895   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5896   op_cost(0);
 5897   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5898   interface(MEMORY_INTER) %{
 5899     base($reg);
 5900     index($lreg);
 5901     scale($scale);
 5902     disp(0x0);
 5903   %}
 5904 %}
 5905 
 5906 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5907 %{
 5908   predicate(CompressedOops::shift() == 0);
 5909   constraint(ALLOC_IN_RC(ptr_reg));
 5910   match(AddP (DecodeN reg) (ConvI2L ireg));
 5911   op_cost(0);
 5912   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5913   interface(MEMORY_INTER) %{
 5914     base($reg);
 5915     index($ireg);
 5916     scale(0x0);
 5917     disp(0x0);
 5918   %}
 5919 %}
 5920 
 5921 operand indIndexN(iRegN reg, iRegL lreg)
 5922 %{
 5923   predicate(CompressedOops::shift() == 0);
 5924   constraint(ALLOC_IN_RC(ptr_reg));
 5925   match(AddP (DecodeN reg) lreg);
 5926   op_cost(0);
 5927   format %{ "$reg, $lreg\t# narrow" %}
 5928   interface(MEMORY_INTER) %{
 5929     base($reg);
 5930     index($lreg);
 5931     scale(0x0);
 5932     disp(0x0);
 5933   %}
 5934 %}
 5935 
 5936 operand indOffIN(iRegN reg, immIOffset off)
 5937 %{
 5938   predicate(CompressedOops::shift() == 0);
 5939   constraint(ALLOC_IN_RC(ptr_reg));
 5940   match(AddP (DecodeN reg) off);
 5941   op_cost(0);
 5942   format %{ "[$reg, $off]\t# narrow" %}
 5943   interface(MEMORY_INTER) %{
 5944     base($reg);
 5945     index(0xffffffff);
 5946     scale(0x0);
 5947     disp($off);
 5948   %}
 5949 %}
 5950 
 5951 operand indOffLN(iRegN reg, immLoffset off)
 5952 %{
 5953   predicate(CompressedOops::shift() == 0);
 5954   constraint(ALLOC_IN_RC(ptr_reg));
 5955   match(AddP (DecodeN reg) off);
 5956   op_cost(0);
 5957   format %{ "[$reg, $off]\t# narrow" %}
 5958   interface(MEMORY_INTER) %{
 5959     base($reg);
 5960     index(0xffffffff);
 5961     scale(0x0);
 5962     disp($off);
 5963   %}
 5964 %}
 5965 
 5966 
 5967 
 5968 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5969 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 5970 %{
 5971   constraint(ALLOC_IN_RC(ptr_reg));
 5972   match(AddP reg off);
 5973   op_cost(0);
 5974   format %{ "[$reg, $off]" %}
 5975   interface(MEMORY_INTER) %{
 5976     base($reg);
 5977     index(0xffffffff);
 5978     scale(0x0);
 5979     disp($off);
 5980   %}
 5981 %}
 5982 
 5983 //----------Special Memory Operands--------------------------------------------
 5984 // Stack Slot Operand - This operand is used for loading and storing temporary
 5985 //                      values on the stack where a match requires a value to
 5986 //                      flow through memory.
 5987 operand stackSlotP(sRegP reg)
 5988 %{
 5989   constraint(ALLOC_IN_RC(stack_slots));
 5990   op_cost(100);
 5991   // No match rule because this operand is only generated in matching
 5992   // match(RegP);
 5993   format %{ "[$reg]" %}
 5994   interface(MEMORY_INTER) %{
 5995     base(0x1e);  // RSP
 5996     index(0x0);  // No Index
 5997     scale(0x0);  // No Scale
 5998     disp($reg);  // Stack Offset
 5999   %}
 6000 %}
 6001 
 6002 operand stackSlotI(sRegI reg)
 6003 %{
 6004   constraint(ALLOC_IN_RC(stack_slots));
 6005   // No match rule because this operand is only generated in matching
 6006   // match(RegI);
 6007   format %{ "[$reg]" %}
 6008   interface(MEMORY_INTER) %{
 6009     base(0x1e);  // RSP
 6010     index(0x0);  // No Index
 6011     scale(0x0);  // No Scale
 6012     disp($reg);  // Stack Offset
 6013   %}
 6014 %}
 6015 
 6016 operand stackSlotF(sRegF reg)
 6017 %{
 6018   constraint(ALLOC_IN_RC(stack_slots));
 6019   // No match rule because this operand is only generated in matching
 6020   // match(RegF);
 6021   format %{ "[$reg]" %}
 6022   interface(MEMORY_INTER) %{
 6023     base(0x1e);  // RSP
 6024     index(0x0);  // No Index
 6025     scale(0x0);  // No Scale
 6026     disp($reg);  // Stack Offset
 6027   %}
 6028 %}
 6029 
 6030 operand stackSlotD(sRegD reg)
 6031 %{
 6032   constraint(ALLOC_IN_RC(stack_slots));
 6033   // No match rule because this operand is only generated in matching
 6034   // match(RegD);
 6035   format %{ "[$reg]" %}
 6036   interface(MEMORY_INTER) %{
 6037     base(0x1e);  // RSP
 6038     index(0x0);  // No Index
 6039     scale(0x0);  // No Scale
 6040     disp($reg);  // Stack Offset
 6041   %}
 6042 %}
 6043 
 6044 operand stackSlotL(sRegL reg)
 6045 %{
 6046   constraint(ALLOC_IN_RC(stack_slots));
 6047   // No match rule because this operand is only generated in matching
 6048   // match(RegL);
 6049   format %{ "[$reg]" %}
 6050   interface(MEMORY_INTER) %{
 6051     base(0x1e);  // RSP
 6052     index(0x0);  // No Index
 6053     scale(0x0);  // No Scale
 6054     disp($reg);  // Stack Offset
 6055   %}
 6056 %}
 6057 
 6058 // Operands for expressing Control Flow
 6059 // NOTE: Label is a predefined operand which should not be redefined in
 6060 //       the AD file. It is generically handled within the ADLC.
 6061 
 6062 //----------Conditional Branch Operands----------------------------------------
 6063 // Comparison Op  - This is the operation of the comparison, and is limited to
 6064 //                  the following set of codes:
 6065 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6066 //
 6067 // Other attributes of the comparison, such as unsignedness, are specified
 6068 // by the comparison instruction that sets a condition code flags register.
 6069 // That result is represented by a flags operand whose subtype is appropriate
 6070 // to the unsignedness (etc.) of the comparison.
 6071 //
 6072 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6073 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6074 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6075 
 6076 // used for signed integral comparisons and fp comparisons
 6077 
 6078 operand cmpOp()
 6079 %{
 6080   match(Bool);
 6081 
 6082   format %{ "" %}
 6083   interface(COND_INTER) %{
 6084     equal(0x0, "eq");
 6085     not_equal(0x1, "ne");
 6086     less(0xb, "lt");
 6087     greater_equal(0xa, "ge");
 6088     less_equal(0xd, "le");
 6089     greater(0xc, "gt");
 6090     overflow(0x6, "vs");
 6091     no_overflow(0x7, "vc");
 6092   %}
 6093 %}
 6094 
 6095 // used for unsigned integral comparisons
 6096 
 6097 operand cmpOpU()
 6098 %{
 6099   match(Bool);
 6100 
 6101   format %{ "" %}
 6102   interface(COND_INTER) %{
 6103     equal(0x0, "eq");
 6104     not_equal(0x1, "ne");
 6105     less(0x3, "lo");
 6106     greater_equal(0x2, "hs");
 6107     less_equal(0x9, "ls");
 6108     greater(0x8, "hi");
 6109     overflow(0x6, "vs");
 6110     no_overflow(0x7, "vc");
 6111   %}
 6112 %}
 6113 
 6114 // used for certain integral comparisons which can be
 6115 // converted to cbxx or tbxx instructions
 6116 
 6117 operand cmpOpEqNe()
 6118 %{
 6119   match(Bool);
 6120   op_cost(0);
 6121   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6122             || n->as_Bool()->_test._test == BoolTest::eq);
 6123 
 6124   format %{ "" %}
 6125   interface(COND_INTER) %{
 6126     equal(0x0, "eq");
 6127     not_equal(0x1, "ne");
 6128     less(0xb, "lt");
 6129     greater_equal(0xa, "ge");
 6130     less_equal(0xd, "le");
 6131     greater(0xc, "gt");
 6132     overflow(0x6, "vs");
 6133     no_overflow(0x7, "vc");
 6134   %}
 6135 %}
 6136 
 6137 // used for certain integral comparisons which can be
 6138 // converted to cbxx or tbxx instructions
 6139 
 6140 operand cmpOpLtGe()
 6141 %{
 6142   match(Bool);
 6143   op_cost(0);
 6144 
 6145   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6146             || n->as_Bool()->_test._test == BoolTest::ge);
 6147 
 6148   format %{ "" %}
 6149   interface(COND_INTER) %{
 6150     equal(0x0, "eq");
 6151     not_equal(0x1, "ne");
 6152     less(0xb, "lt");
 6153     greater_equal(0xa, "ge");
 6154     less_equal(0xd, "le");
 6155     greater(0xc, "gt");
 6156     overflow(0x6, "vs");
 6157     no_overflow(0x7, "vc");
 6158   %}
 6159 %}
 6160 
 6161 // used for certain unsigned integral comparisons which can be
 6162 // converted to cbxx or tbxx instructions
 6163 
 6164 operand cmpOpUEqNeLtGe()
 6165 %{
 6166   match(Bool);
 6167   op_cost(0);
 6168 
 6169   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6170             || n->as_Bool()->_test._test == BoolTest::ne
 6171             || n->as_Bool()->_test._test == BoolTest::lt
 6172             || n->as_Bool()->_test._test == BoolTest::ge);
 6173 
 6174   format %{ "" %}
 6175   interface(COND_INTER) %{
 6176     equal(0x0, "eq");
 6177     not_equal(0x1, "ne");
 6178     less(0xb, "lt");
 6179     greater_equal(0xa, "ge");
 6180     less_equal(0xd, "le");
 6181     greater(0xc, "gt");
 6182     overflow(0x6, "vs");
 6183     no_overflow(0x7, "vc");
 6184   %}
 6185 %}
 6186 
 6187 // Special operand allowing long args to int ops to be truncated for free
 6188 
 6189 operand iRegL2I(iRegL reg) %{
 6190 
 6191   op_cost(0);
 6192 
 6193   match(ConvL2I reg);
 6194 
 6195   format %{ "l2i($reg)" %}
 6196 
 6197   interface(REG_INTER)
 6198 %}
 6199 
 6200 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6201 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6202 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6203 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6204 
 6205 //----------OPERAND CLASSES----------------------------------------------------
 6206 // Operand Classes are groups of operands that are used as to simplify
 6207 // instruction definitions by not requiring the AD writer to specify
 6208 // separate instructions for every form of operand when the
 6209 // instruction accepts multiple operand types with the same basic
 6210 // encoding and format. The classic case of this is memory operands.
 6211 
 6212 // memory is used to define read/write location for load/store
 6213 // instruction defs. we can turn a memory op into an Address
 6214 
 6215 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6216                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6217 
 6218 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6219                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6220 
 6221 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6222                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6223 
 6224 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6225                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6226 
 6227 // All of the memory operands. For the pipeline description.
 6228 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6229                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6230                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6231 
 6232 
 6233 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6234 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6235 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6236 // can be elided because the 32-bit instruction will just employ the
 6237 // lower 32 bits anyway.
 6238 //
 6239 // n.b. this does not elide all L2I conversions. if the truncated
 6240 // value is consumed by more than one operation then the ConvL2I
 6241 // cannot be bundled into the consuming nodes so an l2i gets planted
 6242 // (actually a movw $dst $src) and the downstream instructions consume
 6243 // the result of the l2i as an iRegI input. That's a shame since the
 6244 // movw is actually redundant but its not too costly.
 6245 
 6246 opclass iRegIorL2I(iRegI, iRegL2I);
 6247 
 6248 //----------PIPELINE-----------------------------------------------------------
 6249 // Rules which define the behavior of the target architectures pipeline.
 6250 
 6251 // For specific pipelines, eg A53, define the stages of that pipeline
 6252 //pipe_desc(ISS, EX1, EX2, WR);
 6253 #define ISS S0
 6254 #define EX1 S1
 6255 #define EX2 S2
 6256 #define WR  S3
 6257 
 6258 // Integer ALU reg operation
 6259 pipeline %{
 6260 
 6261 attributes %{
 6262   // ARM instructions are of fixed length
 6263   fixed_size_instructions;        // Fixed size instructions TODO does
 6264   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6265   // ARM instructions come in 32-bit word units
 6266   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6267   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6268   instruction_fetch_units = 1;       // of 64 bytes
 6269 
 6270   // List of nop instructions
 6271   nops( MachNop );
 6272 %}
 6273 
 6274 // We don't use an actual pipeline model so don't care about resources
 6275 // or description. we do use pipeline classes to introduce fixed
 6276 // latencies
 6277 
 6278 //----------RESOURCES----------------------------------------------------------
 6279 // Resources are the functional units available to the machine
 6280 
 6281 resources( INS0, INS1, INS01 = INS0 | INS1,
 6282            ALU0, ALU1, ALU = ALU0 | ALU1,
 6283            MAC,
 6284            DIV,
 6285            BRANCH,
 6286            LDST,
 6287            NEON_FP);
 6288 
 6289 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6290 // Pipeline Description specifies the stages in the machine's pipeline
 6291 
 6292 // Define the pipeline as a generic 6 stage pipeline
 6293 pipe_desc(S0, S1, S2, S3, S4, S5);
 6294 
 6295 //----------PIPELINE CLASSES---------------------------------------------------
 6296 // Pipeline Classes describe the stages in which input and output are
 6297 // referenced by the hardware pipeline.
 6298 
 6299 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6300 %{
 6301   single_instruction;
 6302   src1   : S1(read);
 6303   src2   : S2(read);
 6304   dst    : S5(write);
 6305   INS01  : ISS;
 6306   NEON_FP : S5;
 6307 %}
 6308 
 6309 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6310 %{
 6311   single_instruction;
 6312   src1   : S1(read);
 6313   src2   : S2(read);
 6314   dst    : S5(write);
 6315   INS01  : ISS;
 6316   NEON_FP : S5;
 6317 %}
 6318 
 6319 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6320 %{
 6321   single_instruction;
 6322   src    : S1(read);
 6323   dst    : S5(write);
 6324   INS01  : ISS;
 6325   NEON_FP : S5;
 6326 %}
 6327 
 6328 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6329 %{
 6330   single_instruction;
 6331   src    : S1(read);
 6332   dst    : S5(write);
 6333   INS01  : ISS;
 6334   NEON_FP : S5;
 6335 %}
 6336 
 6337 pipe_class fp_d2f(vRegF dst, vRegD src)
 6338 %{
 6339   single_instruction;
 6340   src    : S1(read);
 6341   dst    : S5(write);
 6342   INS01  : ISS;
 6343   NEON_FP : S5;
 6344 %}
 6345 
 6346 pipe_class fp_f2d(vRegD dst, vRegF src)
 6347 %{
 6348   single_instruction;
 6349   src    : S1(read);
 6350   dst    : S5(write);
 6351   INS01  : ISS;
 6352   NEON_FP : S5;
 6353 %}
 6354 
 6355 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6356 %{
 6357   single_instruction;
 6358   src    : S1(read);
 6359   dst    : S5(write);
 6360   INS01  : ISS;
 6361   NEON_FP : S5;
 6362 %}
 6363 
 6364 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6365 %{
 6366   single_instruction;
 6367   src    : S1(read);
 6368   dst    : S5(write);
 6369   INS01  : ISS;
 6370   NEON_FP : S5;
 6371 %}
 6372 
 6373 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6374 %{
 6375   single_instruction;
 6376   src    : S1(read);
 6377   dst    : S5(write);
 6378   INS01  : ISS;
 6379   NEON_FP : S5;
 6380 %}
 6381 
 6382 pipe_class fp_l2f(vRegF dst, iRegL src)
 6383 %{
 6384   single_instruction;
 6385   src    : S1(read);
 6386   dst    : S5(write);
 6387   INS01  : ISS;
 6388   NEON_FP : S5;
 6389 %}
 6390 
 6391 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6392 %{
 6393   single_instruction;
 6394   src    : S1(read);
 6395   dst    : S5(write);
 6396   INS01  : ISS;
 6397   NEON_FP : S5;
 6398 %}
 6399 
 6400 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6401 %{
 6402   single_instruction;
 6403   src    : S1(read);
 6404   dst    : S5(write);
 6405   INS01  : ISS;
 6406   NEON_FP : S5;
 6407 %}
 6408 
 6409 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6410 %{
 6411   single_instruction;
 6412   src    : S1(read);
 6413   dst    : S5(write);
 6414   INS01  : ISS;
 6415   NEON_FP : S5;
 6416 %}
 6417 
 6418 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6419 %{
 6420   single_instruction;
 6421   src    : S1(read);
 6422   dst    : S5(write);
 6423   INS01  : ISS;
 6424   NEON_FP : S5;
 6425 %}
 6426 
 6427 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6428 %{
 6429   single_instruction;
 6430   src1   : S1(read);
 6431   src2   : S2(read);
 6432   dst    : S5(write);
 6433   INS0   : ISS;
 6434   NEON_FP : S5;
 6435 %}
 6436 
 6437 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6438 %{
 6439   single_instruction;
 6440   src1   : S1(read);
 6441   src2   : S2(read);
 6442   dst    : S5(write);
 6443   INS0   : ISS;
 6444   NEON_FP : S5;
 6445 %}
 6446 
 6447 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6448 %{
 6449   single_instruction;
 6450   cr     : S1(read);
 6451   src1   : S1(read);
 6452   src2   : S1(read);
 6453   dst    : S3(write);
 6454   INS01  : ISS;
 6455   NEON_FP : S3;
 6456 %}
 6457 
 6458 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6459 %{
 6460   single_instruction;
 6461   cr     : S1(read);
 6462   src1   : S1(read);
 6463   src2   : S1(read);
 6464   dst    : S3(write);
 6465   INS01  : ISS;
 6466   NEON_FP : S3;
 6467 %}
 6468 
 6469 pipe_class fp_imm_s(vRegF dst)
 6470 %{
 6471   single_instruction;
 6472   dst    : S3(write);
 6473   INS01  : ISS;
 6474   NEON_FP : S3;
 6475 %}
 6476 
 6477 pipe_class fp_imm_d(vRegD dst)
 6478 %{
 6479   single_instruction;
 6480   dst    : S3(write);
 6481   INS01  : ISS;
 6482   NEON_FP : S3;
 6483 %}
 6484 
 6485 pipe_class fp_load_constant_s(vRegF dst)
 6486 %{
 6487   single_instruction;
 6488   dst    : S4(write);
 6489   INS01  : ISS;
 6490   NEON_FP : S4;
 6491 %}
 6492 
 6493 pipe_class fp_load_constant_d(vRegD dst)
 6494 %{
 6495   single_instruction;
 6496   dst    : S4(write);
 6497   INS01  : ISS;
 6498   NEON_FP : S4;
 6499 %}
 6500 
 6501 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
 6502 %{
 6503   single_instruction;
 6504   dst    : S5(write);
 6505   src1   : S1(read);
 6506   src2   : S1(read);
 6507   INS01  : ISS;
 6508   NEON_FP : S5;
 6509 %}
 6510 
 6511 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
 6512 %{
 6513   single_instruction;
 6514   dst    : S5(write);
 6515   src1   : S1(read);
 6516   src2   : S1(read);
 6517   INS0   : ISS;
 6518   NEON_FP : S5;
 6519 %}
 6520 
 6521 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
 6522 %{
 6523   single_instruction;
 6524   dst    : S5(write);
 6525   src1   : S1(read);
 6526   src2   : S1(read);
 6527   dst    : S1(read);
 6528   INS01  : ISS;
 6529   NEON_FP : S5;
 6530 %}
 6531 
 6532 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
 6533 %{
 6534   single_instruction;
 6535   dst    : S5(write);
 6536   src1   : S1(read);
 6537   src2   : S1(read);
 6538   dst    : S1(read);
 6539   INS0   : ISS;
 6540   NEON_FP : S5;
 6541 %}
 6542 
 6543 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
 6544 %{
 6545   single_instruction;
 6546   dst    : S4(write);
 6547   src1   : S2(read);
 6548   src2   : S2(read);
 6549   INS01  : ISS;
 6550   NEON_FP : S4;
 6551 %}
 6552 
 6553 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
 6554 %{
 6555   single_instruction;
 6556   dst    : S4(write);
 6557   src1   : S2(read);
 6558   src2   : S2(read);
 6559   INS0   : ISS;
 6560   NEON_FP : S4;
 6561 %}
 6562 
 6563 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
 6564 %{
 6565   single_instruction;
 6566   dst    : S3(write);
 6567   src1   : S2(read);
 6568   src2   : S2(read);
 6569   INS01  : ISS;
 6570   NEON_FP : S3;
 6571 %}
 6572 
 6573 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
 6574 %{
 6575   single_instruction;
 6576   dst    : S3(write);
 6577   src1   : S2(read);
 6578   src2   : S2(read);
 6579   INS0   : ISS;
 6580   NEON_FP : S3;
 6581 %}
 6582 
 6583 pipe_class vshift64(vecD dst, vecD src, vecX shift)
 6584 %{
 6585   single_instruction;
 6586   dst    : S3(write);
 6587   src    : S1(read);
 6588   shift  : S1(read);
 6589   INS01  : ISS;
 6590   NEON_FP : S3;
 6591 %}
 6592 
 6593 pipe_class vshift128(vecX dst, vecX src, vecX shift)
 6594 %{
 6595   single_instruction;
 6596   dst    : S3(write);
 6597   src    : S1(read);
 6598   shift  : S1(read);
 6599   INS0   : ISS;
 6600   NEON_FP : S3;
 6601 %}
 6602 
 6603 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
 6604 %{
 6605   single_instruction;
 6606   dst    : S3(write);
 6607   src    : S1(read);
 6608   INS01  : ISS;
 6609   NEON_FP : S3;
 6610 %}
 6611 
 6612 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
 6613 %{
 6614   single_instruction;
 6615   dst    : S3(write);
 6616   src    : S1(read);
 6617   INS0   : ISS;
 6618   NEON_FP : S3;
 6619 %}
 6620 
 6621 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
 6622 %{
 6623   single_instruction;
 6624   dst    : S5(write);
 6625   src1   : S1(read);
 6626   src2   : S1(read);
 6627   INS01  : ISS;
 6628   NEON_FP : S5;
 6629 %}
 6630 
 6631 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
 6632 %{
 6633   single_instruction;
 6634   dst    : S5(write);
 6635   src1   : S1(read);
 6636   src2   : S1(read);
 6637   INS0   : ISS;
 6638   NEON_FP : S5;
 6639 %}
 6640 
 6641 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
 6642 %{
 6643   single_instruction;
 6644   dst    : S5(write);
 6645   src1   : S1(read);
 6646   src2   : S1(read);
 6647   INS0   : ISS;
 6648   NEON_FP : S5;
 6649 %}
 6650 
 6651 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
 6652 %{
 6653   single_instruction;
 6654   dst    : S5(write);
 6655   src1   : S1(read);
 6656   src2   : S1(read);
 6657   INS0   : ISS;
 6658   NEON_FP : S5;
 6659 %}
 6660 
 6661 pipe_class vsqrt_fp128(vecX dst, vecX src)
 6662 %{
 6663   single_instruction;
 6664   dst    : S5(write);
 6665   src    : S1(read);
 6666   INS0   : ISS;
 6667   NEON_FP : S5;
 6668 %}
 6669 
 6670 pipe_class vunop_fp64(vecD dst, vecD src)
 6671 %{
 6672   single_instruction;
 6673   dst    : S5(write);
 6674   src    : S1(read);
 6675   INS01  : ISS;
 6676   NEON_FP : S5;
 6677 %}
 6678 
 6679 pipe_class vunop_fp128(vecX dst, vecX src)
 6680 %{
 6681   single_instruction;
 6682   dst    : S5(write);
 6683   src    : S1(read);
 6684   INS0   : ISS;
 6685   NEON_FP : S5;
 6686 %}
 6687 
 6688 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
 6689 %{
 6690   single_instruction;
 6691   dst    : S3(write);
 6692   src    : S1(read);
 6693   INS01  : ISS;
 6694   NEON_FP : S3;
 6695 %}
 6696 
 6697 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
 6698 %{
 6699   single_instruction;
 6700   dst    : S3(write);
 6701   src    : S1(read);
 6702   INS01  : ISS;
 6703   NEON_FP : S3;
 6704 %}
 6705 
 6706 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
 6707 %{
 6708   single_instruction;
 6709   dst    : S3(write);
 6710   src    : S1(read);
 6711   INS01  : ISS;
 6712   NEON_FP : S3;
 6713 %}
 6714 
 6715 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
 6716 %{
 6717   single_instruction;
 6718   dst    : S3(write);
 6719   src    : S1(read);
 6720   INS01  : ISS;
 6721   NEON_FP : S3;
 6722 %}
 6723 
 6724 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
 6725 %{
 6726   single_instruction;
 6727   dst    : S3(write);
 6728   src    : S1(read);
 6729   INS01  : ISS;
 6730   NEON_FP : S3;
 6731 %}
 6732 
 6733 pipe_class vmovi_reg_imm64(vecD dst)
 6734 %{
 6735   single_instruction;
 6736   dst    : S3(write);
 6737   INS01  : ISS;
 6738   NEON_FP : S3;
 6739 %}
 6740 
 6741 pipe_class vmovi_reg_imm128(vecX dst)
 6742 %{
 6743   single_instruction;
 6744   dst    : S3(write);
 6745   INS0   : ISS;
 6746   NEON_FP : S3;
 6747 %}
 6748 
 6749 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
 6750 %{
 6751   single_instruction;
 6752   dst    : S5(write);
 6753   mem    : ISS(read);
 6754   INS01  : ISS;
 6755   NEON_FP : S3;
 6756 %}
 6757 
 6758 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
 6759 %{
 6760   single_instruction;
 6761   dst    : S5(write);
 6762   mem    : ISS(read);
 6763   INS01  : ISS;
 6764   NEON_FP : S3;
 6765 %}
 6766 
 6767 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
 6768 %{
 6769   single_instruction;
 6770   mem    : ISS(read);
 6771   src    : S2(read);
 6772   INS01  : ISS;
 6773   NEON_FP : S3;
 6774 %}
 6775 
 6776 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
 6777 %{
 6778   single_instruction;
 6779   mem    : ISS(read);
 6780   src    : S2(read);
 6781   INS01  : ISS;
 6782   NEON_FP : S3;
 6783 %}
 6784 
 6785 //------- Integer ALU operations --------------------------
 6786 
 6787 // Integer ALU reg-reg operation
 6788 // Operands needed in EX1, result generated in EX2
 6789 // Eg.  ADD     x0, x1, x2
 6790 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6791 %{
 6792   single_instruction;
 6793   dst    : EX2(write);
 6794   src1   : EX1(read);
 6795   src2   : EX1(read);
 6796   INS01  : ISS; // Dual issue as instruction 0 or 1
 6797   ALU    : EX2;
 6798 %}
 6799 
 6800 // Integer ALU reg-reg operation with constant shift
 6801 // Shifted register must be available in LATE_ISS instead of EX1
 6802 // Eg.  ADD     x0, x1, x2, LSL #2
 6803 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6804 %{
 6805   single_instruction;
 6806   dst    : EX2(write);
 6807   src1   : EX1(read);
 6808   src2   : ISS(read);
 6809   INS01  : ISS;
 6810   ALU    : EX2;
 6811 %}
 6812 
 6813 // Integer ALU reg operation with constant shift
 6814 // Eg.  LSL     x0, x1, #shift
 6815 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6816 %{
 6817   single_instruction;
 6818   dst    : EX2(write);
 6819   src1   : ISS(read);
 6820   INS01  : ISS;
 6821   ALU    : EX2;
 6822 %}
 6823 
 6824 // Integer ALU reg-reg operation with variable shift
 6825 // Both operands must be available in LATE_ISS instead of EX1
 6826 // Result is available in EX1 instead of EX2
 6827 // Eg.  LSLV    x0, x1, x2
 6828 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6829 %{
 6830   single_instruction;
 6831   dst    : EX1(write);
 6832   src1   : ISS(read);
 6833   src2   : ISS(read);
 6834   INS01  : ISS;
 6835   ALU    : EX1;
 6836 %}
 6837 
 6838 // Integer ALU reg-reg operation with extract
 6839 // As for _vshift above, but result generated in EX2
 6840 // Eg.  EXTR    x0, x1, x2, #N
 6841 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6842 %{
 6843   single_instruction;
 6844   dst    : EX2(write);
 6845   src1   : ISS(read);
 6846   src2   : ISS(read);
 6847   INS1   : ISS; // Can only dual issue as Instruction 1
 6848   ALU    : EX1;
 6849 %}
 6850 
 6851 // Integer ALU reg operation
 6852 // Eg.  NEG     x0, x1
 6853 pipe_class ialu_reg(iRegI dst, iRegI src)
 6854 %{
 6855   single_instruction;
 6856   dst    : EX2(write);
 6857   src    : EX1(read);
 6858   INS01  : ISS;
 6859   ALU    : EX2;
 6860 %}
 6861 
 6862 // Integer ALU reg mmediate operation
 6863 // Eg.  ADD     x0, x1, #N
 6864 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6865 %{
 6866   single_instruction;
 6867   dst    : EX2(write);
 6868   src1   : EX1(read);
 6869   INS01  : ISS;
 6870   ALU    : EX2;
 6871 %}
 6872 
 6873 // Integer ALU immediate operation (no source operands)
 6874 // Eg.  MOV     x0, #N
 6875 pipe_class ialu_imm(iRegI dst)
 6876 %{
 6877   single_instruction;
 6878   dst    : EX1(write);
 6879   INS01  : ISS;
 6880   ALU    : EX1;
 6881 %}
 6882 
 6883 //------- Compare operation -------------------------------
 6884 
 6885 // Compare reg-reg
 6886 // Eg.  CMP     x0, x1
 6887 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6888 %{
 6889   single_instruction;
 6890 //  fixed_latency(16);
 6891   cr     : EX2(write);
 6892   op1    : EX1(read);
 6893   op2    : EX1(read);
 6894   INS01  : ISS;
 6895   ALU    : EX2;
 6896 %}
 6897 
 6898 // Compare reg-reg
 6899 // Eg.  CMP     x0, #N
 6900 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6901 %{
 6902   single_instruction;
 6903 //  fixed_latency(16);
 6904   cr     : EX2(write);
 6905   op1    : EX1(read);
 6906   INS01  : ISS;
 6907   ALU    : EX2;
 6908 %}
 6909 
 6910 //------- Conditional instructions ------------------------
 6911 
 6912 // Conditional no operands
 6913 // Eg.  CSINC   x0, zr, zr, <cond>
 6914 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6915 %{
 6916   single_instruction;
 6917   cr     : EX1(read);
 6918   dst    : EX2(write);
 6919   INS01  : ISS;
 6920   ALU    : EX2;
 6921 %}
 6922 
 6923 // Conditional 2 operand
 6924 // EG.  CSEL    X0, X1, X2, <cond>
 6925 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6926 %{
 6927   single_instruction;
 6928   cr     : EX1(read);
 6929   src1   : EX1(read);
 6930   src2   : EX1(read);
 6931   dst    : EX2(write);
 6932   INS01  : ISS;
 6933   ALU    : EX2;
 6934 %}
 6935 
 6936 // Conditional 2 operand
 6937 // EG.  CSEL    X0, X1, X2, <cond>
 6938 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6939 %{
 6940   single_instruction;
 6941   cr     : EX1(read);
 6942   src    : EX1(read);
 6943   dst    : EX2(write);
 6944   INS01  : ISS;
 6945   ALU    : EX2;
 6946 %}
 6947 
 6948 //------- Multiply pipeline operations --------------------
 6949 
 6950 // Multiply reg-reg
 6951 // Eg.  MUL     w0, w1, w2
 6952 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6953 %{
 6954   single_instruction;
 6955   dst    : WR(write);
 6956   src1   : ISS(read);
 6957   src2   : ISS(read);
 6958   INS01  : ISS;
 6959   MAC    : WR;
 6960 %}
 6961 
 6962 // Multiply accumulate
 6963 // Eg.  MADD    w0, w1, w2, w3
 6964 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6965 %{
 6966   single_instruction;
 6967   dst    : WR(write);
 6968   src1   : ISS(read);
 6969   src2   : ISS(read);
 6970   src3   : ISS(read);
 6971   INS01  : ISS;
 6972   MAC    : WR;
 6973 %}
 6974 
 6975 // Eg.  MUL     w0, w1, w2
 6976 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6977 %{
 6978   single_instruction;
 6979   fixed_latency(3); // Maximum latency for 64 bit mul
 6980   dst    : WR(write);
 6981   src1   : ISS(read);
 6982   src2   : ISS(read);
 6983   INS01  : ISS;
 6984   MAC    : WR;
 6985 %}
 6986 
 6987 // Multiply accumulate
 6988 // Eg.  MADD    w0, w1, w2, w3
 6989 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6990 %{
 6991   single_instruction;
 6992   fixed_latency(3); // Maximum latency for 64 bit mul
 6993   dst    : WR(write);
 6994   src1   : ISS(read);
 6995   src2   : ISS(read);
 6996   src3   : ISS(read);
 6997   INS01  : ISS;
 6998   MAC    : WR;
 6999 %}
 7000 
 7001 //------- Divide pipeline operations --------------------
 7002 
 7003 // Eg.  SDIV    w0, w1, w2
 7004 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7005 %{
 7006   single_instruction;
 7007   fixed_latency(8); // Maximum latency for 32 bit divide
 7008   dst    : WR(write);
 7009   src1   : ISS(read);
 7010   src2   : ISS(read);
 7011   INS0   : ISS; // Can only dual issue as instruction 0
 7012   DIV    : WR;
 7013 %}
 7014 
 7015 // Eg.  SDIV    x0, x1, x2
 7016 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 7017 %{
 7018   single_instruction;
 7019   fixed_latency(16); // Maximum latency for 64 bit divide
 7020   dst    : WR(write);
 7021   src1   : ISS(read);
 7022   src2   : ISS(read);
 7023   INS0   : ISS; // Can only dual issue as instruction 0
 7024   DIV    : WR;
 7025 %}
 7026 
 7027 //------- Load pipeline operations ------------------------
 7028 
 7029 // Load - prefetch
 7030 // Eg.  PFRM    <mem>
 7031 pipe_class iload_prefetch(memory mem)
 7032 %{
 7033   single_instruction;
 7034   mem    : ISS(read);
 7035   INS01  : ISS;
 7036   LDST   : WR;
 7037 %}
 7038 
 7039 // Load - reg, mem
 7040 // Eg.  LDR     x0, <mem>
 7041 pipe_class iload_reg_mem(iRegI dst, memory mem)
 7042 %{
 7043   single_instruction;
 7044   dst    : WR(write);
 7045   mem    : ISS(read);
 7046   INS01  : ISS;
 7047   LDST   : WR;
 7048 %}
 7049 
 7050 // Load - reg, reg
 7051 // Eg.  LDR     x0, [sp, x1]
 7052 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 7053 %{
 7054   single_instruction;
 7055   dst    : WR(write);
 7056   src    : ISS(read);
 7057   INS01  : ISS;
 7058   LDST   : WR;
 7059 %}
 7060 
 7061 //------- Store pipeline operations -----------------------
 7062 
 7063 // Store - zr, mem
 7064 // Eg.  STR     zr, <mem>
 7065 pipe_class istore_mem(memory mem)
 7066 %{
 7067   single_instruction;
 7068   mem    : ISS(read);
 7069   INS01  : ISS;
 7070   LDST   : WR;
 7071 %}
 7072 
 7073 // Store - reg, mem
 7074 // Eg.  STR     x0, <mem>
 7075 pipe_class istore_reg_mem(iRegI src, memory mem)
 7076 %{
 7077   single_instruction;
 7078   mem    : ISS(read);
 7079   src    : EX2(read);
 7080   INS01  : ISS;
 7081   LDST   : WR;
 7082 %}
 7083 
 7084 // Store - reg, reg
 7085 // Eg. STR      x0, [sp, x1]
 7086 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 7087 %{
 7088   single_instruction;
 7089   dst    : ISS(read);
 7090   src    : EX2(read);
 7091   INS01  : ISS;
 7092   LDST   : WR;
 7093 %}
 7094 
 7095 //------- Store pipeline operations -----------------------
 7096 
 7097 // Branch
 7098 pipe_class pipe_branch()
 7099 %{
 7100   single_instruction;
 7101   INS01  : ISS;
 7102   BRANCH : EX1;
 7103 %}
 7104 
 7105 // Conditional branch
 7106 pipe_class pipe_branch_cond(rFlagsReg cr)
 7107 %{
 7108   single_instruction;
 7109   cr     : EX1(read);
 7110   INS01  : ISS;
 7111   BRANCH : EX1;
 7112 %}
 7113 
 7114 // Compare & Branch
 7115 // EG.  CBZ/CBNZ
 7116 pipe_class pipe_cmp_branch(iRegI op1)
 7117 %{
 7118   single_instruction;
 7119   op1    : EX1(read);
 7120   INS01  : ISS;
 7121   BRANCH : EX1;
 7122 %}
 7123 
 7124 //------- Synchronisation operations ----------------------
 7125 
 7126 // Any operation requiring serialization.
 7127 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7128 pipe_class pipe_serial()
 7129 %{
 7130   single_instruction;
 7131   force_serialization;
 7132   fixed_latency(16);
 7133   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7134   LDST   : WR;
 7135 %}
 7136 
 7137 // Generic big/slow expanded idiom - also serialized
 7138 pipe_class pipe_slow()
 7139 %{
 7140   instruction_count(10);
 7141   multiple_bundles;
 7142   force_serialization;
 7143   fixed_latency(16);
 7144   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7145   LDST   : WR;
 7146 %}
 7147 
 7148 // Empty pipeline class
 7149 pipe_class pipe_class_empty()
 7150 %{
 7151   single_instruction;
 7152   fixed_latency(0);
 7153 %}
 7154 
 7155 // Default pipeline class.
 7156 pipe_class pipe_class_default()
 7157 %{
 7158   single_instruction;
 7159   fixed_latency(2);
 7160 %}
 7161 
 7162 // Pipeline class for compares.
 7163 pipe_class pipe_class_compare()
 7164 %{
 7165   single_instruction;
 7166   fixed_latency(16);
 7167 %}
 7168 
 7169 // Pipeline class for memory operations.
 7170 pipe_class pipe_class_memory()
 7171 %{
 7172   single_instruction;
 7173   fixed_latency(16);
 7174 %}
 7175 
 7176 // Pipeline class for call.
 7177 pipe_class pipe_class_call()
 7178 %{
 7179   single_instruction;
 7180   fixed_latency(100);
 7181 %}
 7182 
 7183 // Define the class for the Nop node.
 7184 define %{
 7185    MachNop = pipe_class_empty;
 7186 %}
 7187 
 7188 %}
 7189 //----------INSTRUCTIONS-------------------------------------------------------
 7190 //
 7191 // match      -- States which machine-independent subtree may be replaced
 7192 //               by this instruction.
 7193 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7194 //               selection to identify a minimum cost tree of machine
 7195 //               instructions that matches a tree of machine-independent
 7196 //               instructions.
 7197 // format     -- A string providing the disassembly for this instruction.
 7198 //               The value of an instruction's operand may be inserted
 7199 //               by referring to it with a '$' prefix.
 7200 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7201 //               to within an encode class as $primary, $secondary, and $tertiary
 7202 //               rrspectively.  The primary opcode is commonly used to
 7203 //               indicate the type of machine instruction, while secondary
 7204 //               and tertiary are often used for prefix options or addressing
 7205 //               modes.
 7206 // ins_encode -- A list of encode classes with parameters. The encode class
 7207 //               name must have been defined in an 'enc_class' specification
 7208 //               in the encode section of the architecture description.
 7209 
 7210 // ============================================================================
 7211 // Memory (Load/Store) Instructions
 7212 
 7213 // Load Instructions
 7214 
 7215 // Load Byte (8 bit signed)
 7216 instruct loadB(iRegINoSp dst, memory1 mem)
 7217 %{
 7218   match(Set dst (LoadB mem));
 7219   predicate(!needs_acquiring_load(n));
 7220 
 7221   ins_cost(4 * INSN_COST);
 7222   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7223 
 7224   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7225 
 7226   ins_pipe(iload_reg_mem);
 7227 %}
 7228 
 7229 // Load Byte (8 bit signed) into long
 7230 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7231 %{
 7232   match(Set dst (ConvI2L (LoadB mem)));
 7233   predicate(!needs_acquiring_load(n->in(1)));
 7234 
 7235   ins_cost(4 * INSN_COST);
 7236   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7237 
 7238   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7239 
 7240   ins_pipe(iload_reg_mem);
 7241 %}
 7242 
 7243 // Load Byte (8 bit unsigned)
 7244 instruct loadUB(iRegINoSp dst, memory1 mem)
 7245 %{
 7246   match(Set dst (LoadUB mem));
 7247   predicate(!needs_acquiring_load(n));
 7248 
 7249   ins_cost(4 * INSN_COST);
 7250   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7251 
 7252   ins_encode(aarch64_enc_ldrb(dst, mem));
 7253 
 7254   ins_pipe(iload_reg_mem);
 7255 %}
 7256 
 7257 // Load Byte (8 bit unsigned) into long
 7258 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7259 %{
 7260   match(Set dst (ConvI2L (LoadUB mem)));
 7261   predicate(!needs_acquiring_load(n->in(1)));
 7262 
 7263   ins_cost(4 * INSN_COST);
 7264   format %{ "ldrb  $dst, $mem\t# byte" %}
 7265 
 7266   ins_encode(aarch64_enc_ldrb(dst, mem));
 7267 
 7268   ins_pipe(iload_reg_mem);
 7269 %}
 7270 
 7271 // Load Short (16 bit signed)
 7272 instruct loadS(iRegINoSp dst, memory2 mem)
 7273 %{
 7274   match(Set dst (LoadS mem));
 7275   predicate(!needs_acquiring_load(n));
 7276 
 7277   ins_cost(4 * INSN_COST);
 7278   format %{ "ldrshw  $dst, $mem\t# short" %}
 7279 
 7280   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7281 
 7282   ins_pipe(iload_reg_mem);
 7283 %}
 7284 
 7285 // Load Short (16 bit signed) into long
 7286 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7287 %{
 7288   match(Set dst (ConvI2L (LoadS mem)));
 7289   predicate(!needs_acquiring_load(n->in(1)));
 7290 
 7291   ins_cost(4 * INSN_COST);
 7292   format %{ "ldrsh  $dst, $mem\t# short" %}
 7293 
 7294   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7295 
 7296   ins_pipe(iload_reg_mem);
 7297 %}
 7298 
 7299 // Load Char (16 bit unsigned)
 7300 instruct loadUS(iRegINoSp dst, memory2 mem)
 7301 %{
 7302   match(Set dst (LoadUS mem));
 7303   predicate(!needs_acquiring_load(n));
 7304 
 7305   ins_cost(4 * INSN_COST);
 7306   format %{ "ldrh  $dst, $mem\t# short" %}
 7307 
 7308   ins_encode(aarch64_enc_ldrh(dst, mem));
 7309 
 7310   ins_pipe(iload_reg_mem);
 7311 %}
 7312 
 7313 // Load Short/Char (16 bit unsigned) into long
 7314 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7315 %{
 7316   match(Set dst (ConvI2L (LoadUS mem)));
 7317   predicate(!needs_acquiring_load(n->in(1)));
 7318 
 7319   ins_cost(4 * INSN_COST);
 7320   format %{ "ldrh  $dst, $mem\t# short" %}
 7321 
 7322   ins_encode(aarch64_enc_ldrh(dst, mem));
 7323 
 7324   ins_pipe(iload_reg_mem);
 7325 %}
 7326 
 7327 // Load Integer (32 bit signed)
 7328 instruct loadI(iRegINoSp dst, memory4 mem)
 7329 %{
 7330   match(Set dst (LoadI mem));
 7331   predicate(!needs_acquiring_load(n));
 7332 
 7333   ins_cost(4 * INSN_COST);
 7334   format %{ "ldrw  $dst, $mem\t# int" %}
 7335 
 7336   ins_encode(aarch64_enc_ldrw(dst, mem));
 7337 
 7338   ins_pipe(iload_reg_mem);
 7339 %}
 7340 
 7341 // Load Integer (32 bit signed) into long
 7342 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7343 %{
 7344   match(Set dst (ConvI2L (LoadI mem)));
 7345   predicate(!needs_acquiring_load(n->in(1)));
 7346 
 7347   ins_cost(4 * INSN_COST);
 7348   format %{ "ldrsw  $dst, $mem\t# int" %}
 7349 
 7350   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7351 
 7352   ins_pipe(iload_reg_mem);
 7353 %}
 7354 
 7355 // Load Integer (32 bit unsigned) into long
 7356 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7357 %{
 7358   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7359   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7360 
 7361   ins_cost(4 * INSN_COST);
 7362   format %{ "ldrw  $dst, $mem\t# int" %}
 7363 
 7364   ins_encode(aarch64_enc_ldrw(dst, mem));
 7365 
 7366   ins_pipe(iload_reg_mem);
 7367 %}
 7368 
 7369 // Load Long (64 bit signed)
 7370 instruct loadL(iRegLNoSp dst, memory8 mem)
 7371 %{
 7372   match(Set dst (LoadL mem));
 7373   predicate(!needs_acquiring_load(n));
 7374 
 7375   ins_cost(4 * INSN_COST);
 7376   format %{ "ldr  $dst, $mem\t# int" %}
 7377 
 7378   ins_encode(aarch64_enc_ldr(dst, mem));
 7379 
 7380   ins_pipe(iload_reg_mem);
 7381 %}
 7382 
 7383 // Load Range
 7384 instruct loadRange(iRegINoSp dst, memory4 mem)
 7385 %{
 7386   match(Set dst (LoadRange mem));
 7387 
 7388   ins_cost(4 * INSN_COST);
 7389   format %{ "ldrw  $dst, $mem\t# range" %}
 7390 
 7391   ins_encode(aarch64_enc_ldrw(dst, mem));
 7392 
 7393   ins_pipe(iload_reg_mem);
 7394 %}
 7395 
 7396 // Load Pointer
 7397 instruct loadP(iRegPNoSp dst, memory8 mem)
 7398 %{
 7399   match(Set dst (LoadP mem));
 7400   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7401 
 7402   ins_cost(4 * INSN_COST);
 7403   format %{ "ldr  $dst, $mem\t# ptr" %}
 7404 
 7405   ins_encode(aarch64_enc_ldr(dst, mem));
 7406 
 7407   ins_pipe(iload_reg_mem);
 7408 %}
 7409 
 7410 // Load Compressed Pointer
 7411 instruct loadN(iRegNNoSp dst, memory4 mem)
 7412 %{
 7413   match(Set dst (LoadN mem));
 7414   predicate(!needs_acquiring_load(n));
 7415 
 7416   ins_cost(4 * INSN_COST);
 7417   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7418 
 7419   ins_encode(aarch64_enc_ldrw(dst, mem));
 7420 
 7421   ins_pipe(iload_reg_mem);
 7422 %}
 7423 
 7424 // Load Klass Pointer
 7425 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7426 %{
 7427   match(Set dst (LoadKlass mem));
 7428   predicate(!needs_acquiring_load(n));
 7429 
 7430   ins_cost(4 * INSN_COST);
 7431   format %{ "ldr  $dst, $mem\t# class" %}
 7432 
 7433   ins_encode(aarch64_enc_ldr(dst, mem));
 7434 
 7435   ins_pipe(iload_reg_mem);
 7436 %}
 7437 
 7438 // Load Narrow Klass Pointer
 7439 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7440 %{
 7441   match(Set dst (LoadNKlass mem));
 7442   predicate(!needs_acquiring_load(n));
 7443 
 7444   ins_cost(4 * INSN_COST);
 7445   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7446 
 7447   ins_encode(aarch64_enc_ldrw(dst, mem));
 7448 
 7449   ins_pipe(iload_reg_mem);
 7450 %}
 7451 
 7452 // Load Float
 7453 instruct loadF(vRegF dst, memory4 mem)
 7454 %{
 7455   match(Set dst (LoadF mem));
 7456   predicate(!needs_acquiring_load(n));
 7457 
 7458   ins_cost(4 * INSN_COST);
 7459   format %{ "ldrs  $dst, $mem\t# float" %}
 7460 
 7461   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7462 
 7463   ins_pipe(pipe_class_memory);
 7464 %}
 7465 
 7466 // Load Double
 7467 instruct loadD(vRegD dst, memory8 mem)
 7468 %{
 7469   match(Set dst (LoadD mem));
 7470   predicate(!needs_acquiring_load(n));
 7471 
 7472   ins_cost(4 * INSN_COST);
 7473   format %{ "ldrd  $dst, $mem\t# double" %}
 7474 
 7475   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7476 
 7477   ins_pipe(pipe_class_memory);
 7478 %}
 7479 
 7480 
 7481 // Load Int Constant
 7482 instruct loadConI(iRegINoSp dst, immI src)
 7483 %{
 7484   match(Set dst src);
 7485 
 7486   ins_cost(INSN_COST);
 7487   format %{ "mov $dst, $src\t# int" %}
 7488 
 7489   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7490 
 7491   ins_pipe(ialu_imm);
 7492 %}
 7493 
 7494 // Load Long Constant
 7495 instruct loadConL(iRegLNoSp dst, immL src)
 7496 %{
 7497   match(Set dst src);
 7498 
 7499   ins_cost(INSN_COST);
 7500   format %{ "mov $dst, $src\t# long" %}
 7501 
 7502   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7503 
 7504   ins_pipe(ialu_imm);
 7505 %}
 7506 
 7507 // Load Pointer Constant
 7508 
 7509 instruct loadConP(iRegPNoSp dst, immP con)
 7510 %{
 7511   match(Set dst con);
 7512 
 7513   ins_cost(INSN_COST * 4);
 7514   format %{
 7515     "mov  $dst, $con\t# ptr\n\t"
 7516   %}
 7517 
 7518   ins_encode(aarch64_enc_mov_p(dst, con));
 7519 
 7520   ins_pipe(ialu_imm);
 7521 %}
 7522 
 7523 // Load Null Pointer Constant
 7524 
 7525 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7526 %{
 7527   match(Set dst con);
 7528 
 7529   ins_cost(INSN_COST);
 7530   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7531 
 7532   ins_encode(aarch64_enc_mov_p0(dst, con));
 7533 
 7534   ins_pipe(ialu_imm);
 7535 %}
 7536 
 7537 // Load Pointer Constant One
 7538 
 7539 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7540 %{
 7541   match(Set dst con);
 7542 
 7543   ins_cost(INSN_COST);
 7544   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7545 
 7546   ins_encode(aarch64_enc_mov_p1(dst, con));
 7547 
 7548   ins_pipe(ialu_imm);
 7549 %}
 7550 
 7551 // Load Byte Map Base Constant
 7552 
 7553 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7554 %{
 7555   match(Set dst con);
 7556 
 7557   ins_cost(INSN_COST);
 7558   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7559 
 7560   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7561 
 7562   ins_pipe(ialu_imm);
 7563 %}
 7564 
 7565 // Load Narrow Pointer Constant
 7566 
 7567 instruct loadConN(iRegNNoSp dst, immN con)
 7568 %{
 7569   match(Set dst con);
 7570 
 7571   ins_cost(INSN_COST * 4);
 7572   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7573 
 7574   ins_encode(aarch64_enc_mov_n(dst, con));
 7575 
 7576   ins_pipe(ialu_imm);
 7577 %}
 7578 
 7579 // Load Narrow Null Pointer Constant
 7580 
 7581 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7582 %{
 7583   match(Set dst con);
 7584 
 7585   ins_cost(INSN_COST);
 7586   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7587 
 7588   ins_encode(aarch64_enc_mov_n0(dst, con));
 7589 
 7590   ins_pipe(ialu_imm);
 7591 %}
 7592 
 7593 // Load Narrow Klass Constant
 7594 
 7595 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7596 %{
 7597   match(Set dst con);
 7598 
 7599   ins_cost(INSN_COST);
 7600   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7601 
 7602   ins_encode(aarch64_enc_mov_nk(dst, con));
 7603 
 7604   ins_pipe(ialu_imm);
 7605 %}
 7606 
 7607 // Load Packed Float Constant
 7608 
 7609 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7610   match(Set dst con);
 7611   ins_cost(INSN_COST * 4);
 7612   format %{ "fmovs  $dst, $con"%}
 7613   ins_encode %{
 7614     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7615   %}
 7616 
 7617   ins_pipe(fp_imm_s);
 7618 %}
 7619 
 7620 // Load Float Constant
 7621 
 7622 instruct loadConF(vRegF dst, immF con) %{
 7623   match(Set dst con);
 7624 
 7625   ins_cost(INSN_COST * 4);
 7626 
 7627   format %{
 7628     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7629   %}
 7630 
 7631   ins_encode %{
 7632     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7633   %}
 7634 
 7635   ins_pipe(fp_load_constant_s);
 7636 %}
 7637 
 7638 // Load Packed Double Constant
 7639 
 7640 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7641   match(Set dst con);
 7642   ins_cost(INSN_COST);
 7643   format %{ "fmovd  $dst, $con"%}
 7644   ins_encode %{
 7645     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7646   %}
 7647 
 7648   ins_pipe(fp_imm_d);
 7649 %}
 7650 
 7651 // Load Double Constant
 7652 
 7653 instruct loadConD(vRegD dst, immD con) %{
 7654   match(Set dst con);
 7655 
 7656   ins_cost(INSN_COST * 5);
 7657   format %{
 7658     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7659   %}
 7660 
 7661   ins_encode %{
 7662     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7663   %}
 7664 
 7665   ins_pipe(fp_load_constant_d);
 7666 %}
 7667 
 7668 // Store Instructions
 7669 
 7670 // Store CMS card-mark Immediate
 7671 instruct storeimmCM0(immI0 zero, memory1 mem)
 7672 %{
 7673   match(Set mem (StoreCM mem zero));
 7674 
 7675   ins_cost(INSN_COST);
 7676   format %{ "storestore (elided)\n\t"
 7677             "strb zr, $mem\t# byte" %}
 7678 
 7679   ins_encode(aarch64_enc_strb0(mem));
 7680 
 7681   ins_pipe(istore_mem);
 7682 %}
 7683 
 7684 // Store CMS card-mark Immediate with intervening StoreStore
 7685 // needed when using CMS with no conditional card marking
 7686 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7687 %{
 7688   match(Set mem (StoreCM mem zero));
 7689 
 7690   ins_cost(INSN_COST * 2);
 7691   format %{ "storestore\n\t"
 7692             "dmb ishst"
 7693             "\n\tstrb zr, $mem\t# byte" %}
 7694 
 7695   ins_encode(aarch64_enc_strb0_ordered(mem));
 7696 
 7697   ins_pipe(istore_mem);
 7698 %}
 7699 
 7700 // Store Byte
 7701 instruct storeB(iRegIorL2I src, memory1 mem)
 7702 %{
 7703   match(Set mem (StoreB mem src));
 7704   predicate(!needs_releasing_store(n));
 7705 
 7706   ins_cost(INSN_COST);
 7707   format %{ "strb  $src, $mem\t# byte" %}
 7708 
 7709   ins_encode(aarch64_enc_strb(src, mem));
 7710 
 7711   ins_pipe(istore_reg_mem);
 7712 %}
 7713 
 7714 
 7715 instruct storeimmB0(immI0 zero, memory1 mem)
 7716 %{
 7717   match(Set mem (StoreB mem zero));
 7718   predicate(!needs_releasing_store(n));
 7719 
 7720   ins_cost(INSN_COST);
 7721   format %{ "strb rscractch2, $mem\t# byte" %}
 7722 
 7723   ins_encode(aarch64_enc_strb0(mem));
 7724 
 7725   ins_pipe(istore_mem);
 7726 %}
 7727 
 7728 // Store Char/Short
 7729 instruct storeC(iRegIorL2I src, memory2 mem)
 7730 %{
 7731   match(Set mem (StoreC mem src));
 7732   predicate(!needs_releasing_store(n));
 7733 
 7734   ins_cost(INSN_COST);
 7735   format %{ "strh  $src, $mem\t# short" %}
 7736 
 7737   ins_encode(aarch64_enc_strh(src, mem));
 7738 
 7739   ins_pipe(istore_reg_mem);
 7740 %}
 7741 
 7742 instruct storeimmC0(immI0 zero, memory2 mem)
 7743 %{
 7744   match(Set mem (StoreC mem zero));
 7745   predicate(!needs_releasing_store(n));
 7746 
 7747   ins_cost(INSN_COST);
 7748   format %{ "strh  zr, $mem\t# short" %}
 7749 
 7750   ins_encode(aarch64_enc_strh0(mem));
 7751 
 7752   ins_pipe(istore_mem);
 7753 %}
 7754 
 7755 // Store Integer
 7756 
 7757 instruct storeI(iRegIorL2I src, memory4 mem)
 7758 %{
 7759   match(Set mem(StoreI mem src));
 7760   predicate(!needs_releasing_store(n));
 7761 
 7762   ins_cost(INSN_COST);
 7763   format %{ "strw  $src, $mem\t# int" %}
 7764 
 7765   ins_encode(aarch64_enc_strw(src, mem));
 7766 
 7767   ins_pipe(istore_reg_mem);
 7768 %}
 7769 
 7770 instruct storeimmI0(immI0 zero, memory4 mem)
 7771 %{
 7772   match(Set mem(StoreI mem zero));
 7773   predicate(!needs_releasing_store(n));
 7774 
 7775   ins_cost(INSN_COST);
 7776   format %{ "strw  zr, $mem\t# int" %}
 7777 
 7778   ins_encode(aarch64_enc_strw0(mem));
 7779 
 7780   ins_pipe(istore_mem);
 7781 %}
 7782 
 7783 // Store Long (64 bit signed)
 7784 instruct storeL(iRegL src, memory8 mem)
 7785 %{
 7786   match(Set mem (StoreL mem src));
 7787   predicate(!needs_releasing_store(n));
 7788 
 7789   ins_cost(INSN_COST);
 7790   format %{ "str  $src, $mem\t# int" %}
 7791 
 7792   ins_encode(aarch64_enc_str(src, mem));
 7793 
 7794   ins_pipe(istore_reg_mem);
 7795 %}
 7796 
 7797 // Store Long (64 bit signed)
 7798 instruct storeimmL0(immL0 zero, memory8 mem)
 7799 %{
 7800   match(Set mem (StoreL mem zero));
 7801   predicate(!needs_releasing_store(n));
 7802 
 7803   ins_cost(INSN_COST);
 7804   format %{ "str  zr, $mem\t# int" %}
 7805 
 7806   ins_encode(aarch64_enc_str0(mem));
 7807 
 7808   ins_pipe(istore_mem);
 7809 %}
 7810 
 7811 // Store Pointer
 7812 instruct storeP(iRegP src, memory8 mem)
 7813 %{
 7814   match(Set mem (StoreP mem src));
 7815   predicate(!needs_releasing_store(n));
 7816 
 7817   ins_cost(INSN_COST);
 7818   format %{ "str  $src, $mem\t# ptr" %}
 7819 
 7820   ins_encode(aarch64_enc_str(src, mem));
 7821 
 7822   ins_pipe(istore_reg_mem);
 7823 %}
 7824 
 7825 // Store Pointer
 7826 instruct storeimmP0(immP0 zero, memory8 mem)
 7827 %{
 7828   match(Set mem (StoreP mem zero));
 7829   predicate(!needs_releasing_store(n));
 7830 
 7831   ins_cost(INSN_COST);
 7832   format %{ "str zr, $mem\t# ptr" %}
 7833 
 7834   ins_encode(aarch64_enc_str0(mem));
 7835 
 7836   ins_pipe(istore_mem);
 7837 %}
 7838 
 7839 // Store Compressed Pointer
 7840 instruct storeN(iRegN src, memory4 mem)
 7841 %{
 7842   match(Set mem (StoreN mem src));
 7843   predicate(!needs_releasing_store(n));
 7844 
 7845   ins_cost(INSN_COST);
 7846   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7847 
 7848   ins_encode(aarch64_enc_strw(src, mem));
 7849 
 7850   ins_pipe(istore_reg_mem);
 7851 %}
 7852 
 7853 instruct storeImmN0(immN0 zero, memory4 mem)
 7854 %{
 7855   match(Set mem (StoreN mem zero));
 7856   predicate(!needs_releasing_store(n));
 7857 
 7858   ins_cost(INSN_COST);
 7859   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7860 
 7861   ins_encode(aarch64_enc_strw0(mem));
 7862 
 7863   ins_pipe(istore_mem);
 7864 %}
 7865 
 7866 // Store Float
 7867 instruct storeF(vRegF src, memory4 mem)
 7868 %{
 7869   match(Set mem (StoreF mem src));
 7870   predicate(!needs_releasing_store(n));
 7871 
 7872   ins_cost(INSN_COST);
 7873   format %{ "strs  $src, $mem\t# float" %}
 7874 
 7875   ins_encode( aarch64_enc_strs(src, mem) );
 7876 
 7877   ins_pipe(pipe_class_memory);
 7878 %}
 7879 
 7880 // TODO
 7881 // implement storeImmF0 and storeFImmPacked
 7882 
 7883 // Store Double
 7884 instruct storeD(vRegD src, memory8 mem)
 7885 %{
 7886   match(Set mem (StoreD mem src));
 7887   predicate(!needs_releasing_store(n));
 7888 
 7889   ins_cost(INSN_COST);
 7890   format %{ "strd  $src, $mem\t# double" %}
 7891 
 7892   ins_encode( aarch64_enc_strd(src, mem) );
 7893 
 7894   ins_pipe(pipe_class_memory);
 7895 %}
 7896 
 7897 // Store Compressed Klass Pointer
 7898 instruct storeNKlass(iRegN src, memory4 mem)
 7899 %{
 7900   predicate(!needs_releasing_store(n));
 7901   match(Set mem (StoreNKlass mem src));
 7902 
 7903   ins_cost(INSN_COST);
 7904   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7905 
 7906   ins_encode(aarch64_enc_strw(src, mem));
 7907 
 7908   ins_pipe(istore_reg_mem);
 7909 %}
 7910 
 7911 // TODO
 7912 // implement storeImmD0 and storeDImmPacked
 7913 
 7914 // prefetch instructions
 7915 // Must be safe to execute with invalid address (cannot fault).
 7916 
 7917 instruct prefetchalloc( memory8 mem ) %{
 7918   match(PrefetchAllocation mem);
 7919 
 7920   ins_cost(INSN_COST);
 7921   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7922 
 7923   ins_encode( aarch64_enc_prefetchw(mem) );
 7924 
 7925   ins_pipe(iload_prefetch);
 7926 %}
 7927 
 7928 //  ---------------- volatile loads and stores ----------------
 7929 
 7930 // Load Byte (8 bit signed)
 7931 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7932 %{
 7933   match(Set dst (LoadB mem));
 7934 
 7935   ins_cost(VOLATILE_REF_COST);
 7936   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7937 
 7938   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7939 
 7940   ins_pipe(pipe_serial);
 7941 %}
 7942 
 7943 // Load Byte (8 bit signed) into long
 7944 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7945 %{
 7946   match(Set dst (ConvI2L (LoadB mem)));
 7947 
 7948   ins_cost(VOLATILE_REF_COST);
 7949   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7950 
 7951   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7952 
 7953   ins_pipe(pipe_serial);
 7954 %}
 7955 
 7956 // Load Byte (8 bit unsigned)
 7957 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7958 %{
 7959   match(Set dst (LoadUB mem));
 7960 
 7961   ins_cost(VOLATILE_REF_COST);
 7962   format %{ "ldarb  $dst, $mem\t# byte" %}
 7963 
 7964   ins_encode(aarch64_enc_ldarb(dst, mem));
 7965 
 7966   ins_pipe(pipe_serial);
 7967 %}
 7968 
 7969 // Load Byte (8 bit unsigned) into long
 7970 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7971 %{
 7972   match(Set dst (ConvI2L (LoadUB mem)));
 7973 
 7974   ins_cost(VOLATILE_REF_COST);
 7975   format %{ "ldarb  $dst, $mem\t# byte" %}
 7976 
 7977   ins_encode(aarch64_enc_ldarb(dst, mem));
 7978 
 7979   ins_pipe(pipe_serial);
 7980 %}
 7981 
 7982 // Load Short (16 bit signed)
 7983 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7984 %{
 7985   match(Set dst (LoadS mem));
 7986 
 7987   ins_cost(VOLATILE_REF_COST);
 7988   format %{ "ldarshw  $dst, $mem\t# short" %}
 7989 
 7990   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7991 
 7992   ins_pipe(pipe_serial);
 7993 %}
 7994 
 7995 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7996 %{
 7997   match(Set dst (LoadUS mem));
 7998 
 7999   ins_cost(VOLATILE_REF_COST);
 8000   format %{ "ldarhw  $dst, $mem\t# short" %}
 8001 
 8002   ins_encode(aarch64_enc_ldarhw(dst, mem));
 8003 
 8004   ins_pipe(pipe_serial);
 8005 %}
 8006 
 8007 // Load Short/Char (16 bit unsigned) into long
 8008 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8009 %{
 8010   match(Set dst (ConvI2L (LoadUS mem)));
 8011 
 8012   ins_cost(VOLATILE_REF_COST);
 8013   format %{ "ldarh  $dst, $mem\t# short" %}
 8014 
 8015   ins_encode(aarch64_enc_ldarh(dst, mem));
 8016 
 8017   ins_pipe(pipe_serial);
 8018 %}
 8019 
 8020 // Load Short/Char (16 bit signed) into long
 8021 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8022 %{
 8023   match(Set dst (ConvI2L (LoadS mem)));
 8024 
 8025   ins_cost(VOLATILE_REF_COST);
 8026   format %{ "ldarh  $dst, $mem\t# short" %}
 8027 
 8028   ins_encode(aarch64_enc_ldarsh(dst, mem));
 8029 
 8030   ins_pipe(pipe_serial);
 8031 %}
 8032 
 8033 // Load Integer (32 bit signed)
 8034 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 8035 %{
 8036   match(Set dst (LoadI mem));
 8037 
 8038   ins_cost(VOLATILE_REF_COST);
 8039   format %{ "ldarw  $dst, $mem\t# int" %}
 8040 
 8041   ins_encode(aarch64_enc_ldarw(dst, mem));
 8042 
 8043   ins_pipe(pipe_serial);
 8044 %}
 8045 
 8046 // Load Integer (32 bit unsigned) into long
 8047 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 8048 %{
 8049   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 8050 
 8051   ins_cost(VOLATILE_REF_COST);
 8052   format %{ "ldarw  $dst, $mem\t# int" %}
 8053 
 8054   ins_encode(aarch64_enc_ldarw(dst, mem));
 8055 
 8056   ins_pipe(pipe_serial);
 8057 %}
 8058 
 8059 // Load Long (64 bit signed)
 8060 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 8061 %{
 8062   match(Set dst (LoadL mem));
 8063 
 8064   ins_cost(VOLATILE_REF_COST);
 8065   format %{ "ldar  $dst, $mem\t# int" %}
 8066 
 8067   ins_encode(aarch64_enc_ldar(dst, mem));
 8068 
 8069   ins_pipe(pipe_serial);
 8070 %}
 8071 
 8072 // Load Pointer
 8073 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 8074 %{
 8075   match(Set dst (LoadP mem));
 8076   predicate(n->as_Load()->barrier_data() == 0);
 8077 
 8078   ins_cost(VOLATILE_REF_COST);
 8079   format %{ "ldar  $dst, $mem\t# ptr" %}
 8080 
 8081   ins_encode(aarch64_enc_ldar(dst, mem));
 8082 
 8083   ins_pipe(pipe_serial);
 8084 %}
 8085 
 8086 // Load Compressed Pointer
 8087 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 8088 %{
 8089   match(Set dst (LoadN mem));
 8090 
 8091   ins_cost(VOLATILE_REF_COST);
 8092   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 8093 
 8094   ins_encode(aarch64_enc_ldarw(dst, mem));
 8095 
 8096   ins_pipe(pipe_serial);
 8097 %}
 8098 
 8099 // Load Float
 8100 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 8101 %{
 8102   match(Set dst (LoadF mem));
 8103 
 8104   ins_cost(VOLATILE_REF_COST);
 8105   format %{ "ldars  $dst, $mem\t# float" %}
 8106 
 8107   ins_encode( aarch64_enc_fldars(dst, mem) );
 8108 
 8109   ins_pipe(pipe_serial);
 8110 %}
 8111 
 8112 // Load Double
 8113 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8114 %{
 8115   match(Set dst (LoadD mem));
 8116 
 8117   ins_cost(VOLATILE_REF_COST);
 8118   format %{ "ldard  $dst, $mem\t# double" %}
 8119 
 8120   ins_encode( aarch64_enc_fldard(dst, mem) );
 8121 
 8122   ins_pipe(pipe_serial);
 8123 %}
 8124 
 8125 // Store Byte
 8126 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8127 %{
 8128   match(Set mem (StoreB mem src));
 8129 
 8130   ins_cost(VOLATILE_REF_COST);
 8131   format %{ "stlrb  $src, $mem\t# byte" %}
 8132 
 8133   ins_encode(aarch64_enc_stlrb(src, mem));
 8134 
 8135   ins_pipe(pipe_class_memory);
 8136 %}
 8137 
 8138 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8139 %{
 8140   match(Set mem (StoreB mem zero));
 8141 
 8142   ins_cost(VOLATILE_REF_COST);
 8143   format %{ "stlrb  zr, $mem\t# byte" %}
 8144 
 8145   ins_encode(aarch64_enc_stlrb0(mem));
 8146 
 8147   ins_pipe(pipe_class_memory);
 8148 %}
 8149 
 8150 // Store Char/Short
 8151 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8152 %{
 8153   match(Set mem (StoreC mem src));
 8154 
 8155   ins_cost(VOLATILE_REF_COST);
 8156   format %{ "stlrh  $src, $mem\t# short" %}
 8157 
 8158   ins_encode(aarch64_enc_stlrh(src, mem));
 8159 
 8160   ins_pipe(pipe_class_memory);
 8161 %}
 8162 
 8163 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8164 %{
 8165   match(Set mem (StoreC mem zero));
 8166 
 8167   ins_cost(VOLATILE_REF_COST);
 8168   format %{ "stlrh  zr, $mem\t# short" %}
 8169 
 8170   ins_encode(aarch64_enc_stlrh0(mem));
 8171 
 8172   ins_pipe(pipe_class_memory);
 8173 %}
 8174 
 8175 // Store Integer
 8176 
 8177 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8178 %{
 8179   match(Set mem(StoreI mem src));
 8180 
 8181   ins_cost(VOLATILE_REF_COST);
 8182   format %{ "stlrw  $src, $mem\t# int" %}
 8183 
 8184   ins_encode(aarch64_enc_stlrw(src, mem));
 8185 
 8186   ins_pipe(pipe_class_memory);
 8187 %}
 8188 
 8189 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8190 %{
 8191   match(Set mem(StoreI mem zero));
 8192 
 8193   ins_cost(VOLATILE_REF_COST);
 8194   format %{ "stlrw  zr, $mem\t# int" %}
 8195 
 8196   ins_encode(aarch64_enc_stlrw0(mem));
 8197 
 8198   ins_pipe(pipe_class_memory);
 8199 %}
 8200 
 8201 // Store Long (64 bit signed)
 8202 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8203 %{
 8204   match(Set mem (StoreL mem src));
 8205 
 8206   ins_cost(VOLATILE_REF_COST);
 8207   format %{ "stlr  $src, $mem\t# int" %}
 8208 
 8209   ins_encode(aarch64_enc_stlr(src, mem));
 8210 
 8211   ins_pipe(pipe_class_memory);
 8212 %}
 8213 
 8214 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8215 %{
 8216   match(Set mem (StoreL mem zero));
 8217 
 8218   ins_cost(VOLATILE_REF_COST);
 8219   format %{ "stlr  zr, $mem\t# int" %}
 8220 
 8221   ins_encode(aarch64_enc_stlr0(mem));
 8222 
 8223   ins_pipe(pipe_class_memory);
 8224 %}
 8225 
 8226 // Store Pointer
 8227 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8228 %{
 8229   match(Set mem (StoreP mem src));
 8230 
 8231   ins_cost(VOLATILE_REF_COST);
 8232   format %{ "stlr  $src, $mem\t# ptr" %}
 8233 
 8234   ins_encode(aarch64_enc_stlr(src, mem));
 8235 
 8236   ins_pipe(pipe_class_memory);
 8237 %}
 8238 
 8239 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8240 %{
 8241   match(Set mem (StoreP mem zero));
 8242 
 8243   ins_cost(VOLATILE_REF_COST);
 8244   format %{ "stlr  zr, $mem\t# ptr" %}
 8245 
 8246   ins_encode(aarch64_enc_stlr0(mem));
 8247 
 8248   ins_pipe(pipe_class_memory);
 8249 %}
 8250 
 8251 // Store Compressed Pointer
 8252 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8253 %{
 8254   match(Set mem (StoreN mem src));
 8255 
 8256   ins_cost(VOLATILE_REF_COST);
 8257   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8258 
 8259   ins_encode(aarch64_enc_stlrw(src, mem));
 8260 
 8261   ins_pipe(pipe_class_memory);
 8262 %}
 8263 
 8264 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8265 %{
 8266   match(Set mem (StoreN mem zero));
 8267 
 8268   ins_cost(VOLATILE_REF_COST);
 8269   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8270 
 8271   ins_encode(aarch64_enc_stlrw0(mem));
 8272 
 8273   ins_pipe(pipe_class_memory);
 8274 %}
 8275 
 8276 // Store Float
 8277 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8278 %{
 8279   match(Set mem (StoreF mem src));
 8280 
 8281   ins_cost(VOLATILE_REF_COST);
 8282   format %{ "stlrs  $src, $mem\t# float" %}
 8283 
 8284   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8285 
 8286   ins_pipe(pipe_class_memory);
 8287 %}
 8288 
 8289 // TODO
 8290 // implement storeImmF0 and storeFImmPacked
 8291 
 8292 // Store Double
 8293 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8294 %{
 8295   match(Set mem (StoreD mem src));
 8296 
 8297   ins_cost(VOLATILE_REF_COST);
 8298   format %{ "stlrd  $src, $mem\t# double" %}
 8299 
 8300   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8301 
 8302   ins_pipe(pipe_class_memory);
 8303 %}
 8304 
 8305 //  ---------------- end of volatile loads and stores ----------------
 8306 
 8307 instruct cacheWB(indirect addr)
 8308 %{
 8309   predicate(VM_Version::supports_data_cache_line_flush());
 8310   match(CacheWB addr);
 8311 
 8312   ins_cost(100);
 8313   format %{"cache wb $addr" %}
 8314   ins_encode %{
 8315     assert($addr->index_position() < 0, "should be");
 8316     assert($addr$$disp == 0, "should be");
 8317     __ cache_wb(Address($addr$$base$$Register, 0));
 8318   %}
 8319   ins_pipe(pipe_slow); // XXX
 8320 %}
 8321 
 8322 instruct cacheWBPreSync()
 8323 %{
 8324   predicate(VM_Version::supports_data_cache_line_flush());
 8325   match(CacheWBPreSync);
 8326 
 8327   ins_cost(100);
 8328   format %{"cache wb presync" %}
 8329   ins_encode %{
 8330     __ cache_wbsync(true);
 8331   %}
 8332   ins_pipe(pipe_slow); // XXX
 8333 %}
 8334 
 8335 instruct cacheWBPostSync()
 8336 %{
 8337   predicate(VM_Version::supports_data_cache_line_flush());
 8338   match(CacheWBPostSync);
 8339 
 8340   ins_cost(100);
 8341   format %{"cache wb postsync" %}
 8342   ins_encode %{
 8343     __ cache_wbsync(false);
 8344   %}
 8345   ins_pipe(pipe_slow); // XXX
 8346 %}
 8347 
 8348 // ============================================================================
 8349 // BSWAP Instructions
 8350 
 8351 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8352   match(Set dst (ReverseBytesI src));
 8353 
 8354   ins_cost(INSN_COST);
 8355   format %{ "revw  $dst, $src" %}
 8356 
 8357   ins_encode %{
 8358     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8359   %}
 8360 
 8361   ins_pipe(ialu_reg);
 8362 %}
 8363 
 8364 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8365   match(Set dst (ReverseBytesL src));
 8366 
 8367   ins_cost(INSN_COST);
 8368   format %{ "rev  $dst, $src" %}
 8369 
 8370   ins_encode %{
 8371     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8372   %}
 8373 
 8374   ins_pipe(ialu_reg);
 8375 %}
 8376 
 8377 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8378   match(Set dst (ReverseBytesUS src));
 8379 
 8380   ins_cost(INSN_COST);
 8381   format %{ "rev16w  $dst, $src" %}
 8382 
 8383   ins_encode %{
 8384     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8385   %}
 8386 
 8387   ins_pipe(ialu_reg);
 8388 %}
 8389 
 8390 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8391   match(Set dst (ReverseBytesS src));
 8392 
 8393   ins_cost(INSN_COST);
 8394   format %{ "rev16w  $dst, $src\n\t"
 8395             "sbfmw $dst, $dst, #0, #15" %}
 8396 
 8397   ins_encode %{
 8398     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8399     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8400   %}
 8401 
 8402   ins_pipe(ialu_reg);
 8403 %}
 8404 
 8405 // ============================================================================
 8406 // Zero Count Instructions
 8407 
 8408 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8409   match(Set dst (CountLeadingZerosI src));
 8410 
 8411   ins_cost(INSN_COST);
 8412   format %{ "clzw  $dst, $src" %}
 8413   ins_encode %{
 8414     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8415   %}
 8416 
 8417   ins_pipe(ialu_reg);
 8418 %}
 8419 
 8420 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8421   match(Set dst (CountLeadingZerosL src));
 8422 
 8423   ins_cost(INSN_COST);
 8424   format %{ "clz   $dst, $src" %}
 8425   ins_encode %{
 8426     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8427   %}
 8428 
 8429   ins_pipe(ialu_reg);
 8430 %}
 8431 
 8432 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8433   match(Set dst (CountTrailingZerosI src));
 8434 
 8435   ins_cost(INSN_COST * 2);
 8436   format %{ "rbitw  $dst, $src\n\t"
 8437             "clzw   $dst, $dst" %}
 8438   ins_encode %{
 8439     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8440     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8441   %}
 8442 
 8443   ins_pipe(ialu_reg);
 8444 %}
 8445 
 8446 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8447   match(Set dst (CountTrailingZerosL src));
 8448 
 8449   ins_cost(INSN_COST * 2);
 8450   format %{ "rbit   $dst, $src\n\t"
 8451             "clz    $dst, $dst" %}
 8452   ins_encode %{
 8453     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8454     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8455   %}
 8456 
 8457   ins_pipe(ialu_reg);
 8458 %}
 8459 
 8460 //---------- Population Count Instructions -------------------------------------
 8461 //
 8462 
 8463 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8464   predicate(UsePopCountInstruction);
 8465   match(Set dst (PopCountI src));
 8466   effect(TEMP tmp);
 8467   ins_cost(INSN_COST * 13);
 8468 
 8469   format %{ "movw   $src, $src\n\t"
 8470             "mov    $tmp, $src\t# vector (1D)\n\t"
 8471             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8472             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8473             "mov    $dst, $tmp\t# vector (1D)" %}
 8474   ins_encode %{
 8475     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8476     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8477     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8478     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8479     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8480   %}
 8481 
 8482   ins_pipe(pipe_class_default);
 8483 %}
 8484 
 8485 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8486   predicate(UsePopCountInstruction);
 8487   match(Set dst (PopCountI (LoadI mem)));
 8488   effect(TEMP tmp);
 8489   ins_cost(INSN_COST * 13);
 8490 
 8491   format %{ "ldrs   $tmp, $mem\n\t"
 8492             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8493             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8494             "mov    $dst, $tmp\t# vector (1D)" %}
 8495   ins_encode %{
 8496     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8497     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8498               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8499     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8500     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8501     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8502   %}
 8503 
 8504   ins_pipe(pipe_class_default);
 8505 %}
 8506 
 8507 // Note: Long.bitCount(long) returns an int.
 8508 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8509   predicate(UsePopCountInstruction);
 8510   match(Set dst (PopCountL src));
 8511   effect(TEMP tmp);
 8512   ins_cost(INSN_COST * 13);
 8513 
 8514   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8515             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8516             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8517             "mov    $dst, $tmp\t# vector (1D)" %}
 8518   ins_encode %{
 8519     __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
 8520     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8521     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8522     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8523   %}
 8524 
 8525   ins_pipe(pipe_class_default);
 8526 %}
 8527 
 8528 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8529   predicate(UsePopCountInstruction);
 8530   match(Set dst (PopCountL (LoadL mem)));
 8531   effect(TEMP tmp);
 8532   ins_cost(INSN_COST * 13);
 8533 
 8534   format %{ "ldrd   $tmp, $mem\n\t"
 8535             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8536             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8537             "mov    $dst, $tmp\t# vector (1D)" %}
 8538   ins_encode %{
 8539     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8540     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8541               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8542     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8543     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8544     __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
 8545   %}
 8546 
 8547   ins_pipe(pipe_class_default);
 8548 %}
 8549 
 8550 // ============================================================================
 8551 // MemBar Instruction
 8552 
 8553 instruct load_fence() %{
 8554   match(LoadFence);
 8555   ins_cost(VOLATILE_REF_COST);
 8556 
 8557   format %{ "load_fence" %}
 8558 
 8559   ins_encode %{
 8560     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8561   %}
 8562   ins_pipe(pipe_serial);
 8563 %}
 8564 
 8565 instruct unnecessary_membar_acquire() %{
 8566   predicate(unnecessary_acquire(n));
 8567   match(MemBarAcquire);
 8568   ins_cost(0);
 8569 
 8570   format %{ "membar_acquire (elided)" %}
 8571 
 8572   ins_encode %{
 8573     __ block_comment("membar_acquire (elided)");
 8574   %}
 8575 
 8576   ins_pipe(pipe_class_empty);
 8577 %}
 8578 
 8579 instruct membar_acquire() %{
 8580   match(MemBarAcquire);
 8581   ins_cost(VOLATILE_REF_COST);
 8582 
 8583   format %{ "membar_acquire\n\t"
 8584             "dmb ish" %}
 8585 
 8586   ins_encode %{
 8587     __ block_comment("membar_acquire");
 8588     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8589   %}
 8590 
 8591   ins_pipe(pipe_serial);
 8592 %}
 8593 
 8594 
 8595 instruct membar_acquire_lock() %{
 8596   match(MemBarAcquireLock);
 8597   ins_cost(VOLATILE_REF_COST);
 8598 
 8599   format %{ "membar_acquire_lock (elided)" %}
 8600 
 8601   ins_encode %{
 8602     __ block_comment("membar_acquire_lock (elided)");
 8603   %}
 8604 
 8605   ins_pipe(pipe_serial);
 8606 %}
 8607 
 8608 instruct store_fence() %{
 8609   match(StoreFence);
 8610   ins_cost(VOLATILE_REF_COST);
 8611 
 8612   format %{ "store_fence" %}
 8613 
 8614   ins_encode %{
 8615     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8616   %}
 8617   ins_pipe(pipe_serial);
 8618 %}
 8619 
 8620 instruct unnecessary_membar_release() %{
 8621   predicate(unnecessary_release(n));
 8622   match(MemBarRelease);
 8623   ins_cost(0);
 8624 
 8625   format %{ "membar_release (elided)" %}
 8626 
 8627   ins_encode %{
 8628     __ block_comment("membar_release (elided)");
 8629   %}
 8630   ins_pipe(pipe_serial);
 8631 %}
 8632 
 8633 instruct membar_release() %{
 8634   match(MemBarRelease);
 8635   ins_cost(VOLATILE_REF_COST);
 8636 
 8637   format %{ "membar_release\n\t"
 8638             "dmb ish" %}
 8639 
 8640   ins_encode %{
 8641     __ block_comment("membar_release");
 8642     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8643   %}
 8644   ins_pipe(pipe_serial);
 8645 %}
 8646 
 8647 instruct membar_storestore() %{
 8648   match(MemBarStoreStore);
 8649   match(StoreStoreFence);
 8650   ins_cost(VOLATILE_REF_COST);
 8651 
 8652   format %{ "MEMBAR-store-store" %}
 8653 
 8654   ins_encode %{
 8655     __ membar(Assembler::StoreStore);
 8656   %}
 8657   ins_pipe(pipe_serial);
 8658 %}
 8659 
 8660 instruct membar_release_lock() %{
 8661   match(MemBarReleaseLock);
 8662   ins_cost(VOLATILE_REF_COST);
 8663 
 8664   format %{ "membar_release_lock (elided)" %}
 8665 
 8666   ins_encode %{
 8667     __ block_comment("membar_release_lock (elided)");
 8668   %}
 8669 
 8670   ins_pipe(pipe_serial);
 8671 %}
 8672 
 8673 instruct unnecessary_membar_volatile() %{
 8674   predicate(unnecessary_volatile(n));
 8675   match(MemBarVolatile);
 8676   ins_cost(0);
 8677 
 8678   format %{ "membar_volatile (elided)" %}
 8679 
 8680   ins_encode %{
 8681     __ block_comment("membar_volatile (elided)");
 8682   %}
 8683 
 8684   ins_pipe(pipe_serial);
 8685 %}
 8686 
 8687 instruct membar_volatile() %{
 8688   match(MemBarVolatile);
 8689   ins_cost(VOLATILE_REF_COST*100);
 8690 
 8691   format %{ "membar_volatile\n\t"
 8692              "dmb ish"%}
 8693 
 8694   ins_encode %{
 8695     __ block_comment("membar_volatile");
 8696     __ membar(Assembler::StoreLoad);
 8697   %}
 8698 
 8699   ins_pipe(pipe_serial);
 8700 %}
 8701 
 8702 // ============================================================================
 8703 // Cast/Convert Instructions
 8704 
 8705 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8706   match(Set dst (CastX2P src));
 8707 
 8708   ins_cost(INSN_COST);
 8709   format %{ "mov $dst, $src\t# long -> ptr" %}
 8710 
 8711   ins_encode %{
 8712     if ($dst$$reg != $src$$reg) {
 8713       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8714     }
 8715   %}
 8716 
 8717   ins_pipe(ialu_reg);
 8718 %}
 8719 
 8720 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8721   match(Set dst (CastP2X src));
 8722 
 8723   ins_cost(INSN_COST);
 8724   format %{ "mov $dst, $src\t# ptr -> long" %}
 8725 
 8726   ins_encode %{
 8727     if ($dst$$reg != $src$$reg) {
 8728       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8729     }
 8730   %}
 8731 
 8732   ins_pipe(ialu_reg);
 8733 %}
 8734 
 8735 // Convert oop into int for vectors alignment masking
 8736 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8737   match(Set dst (ConvL2I (CastP2X src)));
 8738 
 8739   ins_cost(INSN_COST);
 8740   format %{ "movw $dst, $src\t# ptr -> int" %}
 8741   ins_encode %{
 8742     __ movw($dst$$Register, $src$$Register);
 8743   %}
 8744 
 8745   ins_pipe(ialu_reg);
 8746 %}
 8747 
 8748 // Convert compressed oop into int for vectors alignment masking
 8749 // in case of 32bit oops (heap < 4Gb).
 8750 instruct convN2I(iRegINoSp dst, iRegN src)
 8751 %{
 8752   predicate(CompressedOops::shift() == 0);
 8753   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8754 
 8755   ins_cost(INSN_COST);
 8756   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8757   ins_encode %{
 8758     __ movw($dst$$Register, $src$$Register);
 8759   %}
 8760 
 8761   ins_pipe(ialu_reg);
 8762 %}
 8763 
 8764 
 8765 // Convert oop pointer into compressed form
 8766 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8767   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8768   match(Set dst (EncodeP src));
 8769   effect(KILL cr);
 8770   ins_cost(INSN_COST * 3);
 8771   format %{ "encode_heap_oop $dst, $src" %}
 8772   ins_encode %{
 8773     Register s = $src$$Register;
 8774     Register d = $dst$$Register;
 8775     __ encode_heap_oop(d, s);
 8776   %}
 8777   ins_pipe(ialu_reg);
 8778 %}
 8779 
 8780 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8781   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8782   match(Set dst (EncodeP src));
 8783   ins_cost(INSN_COST * 3);
 8784   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8785   ins_encode %{
 8786     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8787   %}
 8788   ins_pipe(ialu_reg);
 8789 %}
 8790 
 8791 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8792   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8793             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8794   match(Set dst (DecodeN src));
 8795   ins_cost(INSN_COST * 3);
 8796   format %{ "decode_heap_oop $dst, $src" %}
 8797   ins_encode %{
 8798     Register s = $src$$Register;
 8799     Register d = $dst$$Register;
 8800     __ decode_heap_oop(d, s);
 8801   %}
 8802   ins_pipe(ialu_reg);
 8803 %}
 8804 
 8805 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8806   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8807             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8808   match(Set dst (DecodeN src));
 8809   ins_cost(INSN_COST * 3);
 8810   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8811   ins_encode %{
 8812     Register s = $src$$Register;
 8813     Register d = $dst$$Register;
 8814     __ decode_heap_oop_not_null(d, s);
 8815   %}
 8816   ins_pipe(ialu_reg);
 8817 %}
 8818 
 8819 // n.b. AArch64 implementations of encode_klass_not_null and
 8820 // decode_klass_not_null do not modify the flags register so, unlike
 8821 // Intel, we don't kill CR as a side effect here
 8822 
 8823 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8824   match(Set dst (EncodePKlass src));
 8825 
 8826   ins_cost(INSN_COST * 3);
 8827   format %{ "encode_klass_not_null $dst,$src" %}
 8828 
 8829   ins_encode %{
 8830     Register src_reg = as_Register($src$$reg);
 8831     Register dst_reg = as_Register($dst$$reg);
 8832     __ encode_klass_not_null(dst_reg, src_reg);
 8833   %}
 8834 
 8835    ins_pipe(ialu_reg);
 8836 %}
 8837 
 8838 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8839   match(Set dst (DecodeNKlass src));
 8840 
 8841   ins_cost(INSN_COST * 3);
 8842   format %{ "decode_klass_not_null $dst,$src" %}
 8843 
 8844   ins_encode %{
 8845     Register src_reg = as_Register($src$$reg);
 8846     Register dst_reg = as_Register($dst$$reg);
 8847     if (dst_reg != src_reg) {
 8848       __ decode_klass_not_null(dst_reg, src_reg);
 8849     } else {
 8850       __ decode_klass_not_null(dst_reg);
 8851     }
 8852   %}
 8853 
 8854    ins_pipe(ialu_reg);
 8855 %}
 8856 
 8857 instruct checkCastPP(iRegPNoSp dst)
 8858 %{
 8859   match(Set dst (CheckCastPP dst));
 8860 
 8861   size(0);
 8862   format %{ "# checkcastPP of $dst" %}
 8863   ins_encode(/* empty encoding */);
 8864   ins_pipe(pipe_class_empty);
 8865 %}
 8866 
 8867 instruct castPP(iRegPNoSp dst)
 8868 %{
 8869   match(Set dst (CastPP dst));
 8870 
 8871   size(0);
 8872   format %{ "# castPP of $dst" %}
 8873   ins_encode(/* empty encoding */);
 8874   ins_pipe(pipe_class_empty);
 8875 %}
 8876 
 8877 instruct castII(iRegI dst)
 8878 %{
 8879   match(Set dst (CastII dst));
 8880 
 8881   size(0);
 8882   format %{ "# castII of $dst" %}
 8883   ins_encode(/* empty encoding */);
 8884   ins_cost(0);
 8885   ins_pipe(pipe_class_empty);
 8886 %}
 8887 
 8888 instruct castLL(iRegL dst)
 8889 %{
 8890   match(Set dst (CastLL dst));
 8891 
 8892   size(0);
 8893   format %{ "# castLL of $dst" %}
 8894   ins_encode(/* empty encoding */);
 8895   ins_cost(0);
 8896   ins_pipe(pipe_class_empty);
 8897 %}
 8898 
 8899 instruct castFF(vRegF dst)
 8900 %{
 8901   match(Set dst (CastFF dst));
 8902 
 8903   size(0);
 8904   format %{ "# castFF of $dst" %}
 8905   ins_encode(/* empty encoding */);
 8906   ins_cost(0);
 8907   ins_pipe(pipe_class_empty);
 8908 %}
 8909 
 8910 instruct castDD(vRegD dst)
 8911 %{
 8912   match(Set dst (CastDD dst));
 8913 
 8914   size(0);
 8915   format %{ "# castDD of $dst" %}
 8916   ins_encode(/* empty encoding */);
 8917   ins_cost(0);
 8918   ins_pipe(pipe_class_empty);
 8919 %}
 8920 
 8921 instruct castVVD(vecD dst)
 8922 %{
 8923   match(Set dst (CastVV dst));
 8924 
 8925   size(0);
 8926   format %{ "# castVV of $dst" %}
 8927   ins_encode(/* empty encoding */);
 8928   ins_cost(0);
 8929   ins_pipe(pipe_class_empty);
 8930 %}
 8931 
 8932 instruct castVVX(vecX dst)
 8933 %{
 8934   match(Set dst (CastVV dst));
 8935 
 8936   size(0);
 8937   format %{ "# castVV of $dst" %}
 8938   ins_encode(/* empty encoding */);
 8939   ins_cost(0);
 8940   ins_pipe(pipe_class_empty);
 8941 %}
 8942 
 8943 instruct castVV(vReg dst)
 8944 %{
 8945   match(Set dst (CastVV dst));
 8946 
 8947   size(0);
 8948   format %{ "# castVV of $dst" %}
 8949   ins_encode(/* empty encoding */);
 8950   ins_cost(0);
 8951   ins_pipe(pipe_class_empty);
 8952 %}
 8953 
 8954 // ============================================================================
 8955 // Atomic operation instructions
 8956 //
 8957 // Intel and SPARC both implement Ideal Node LoadPLocked and
 8958 // Store{PIL}Conditional instructions using a normal load for the
 8959 // LoadPLocked and a CAS for the Store{PIL}Conditional.
 8960 //
 8961 // The ideal code appears only to use LoadPLocked/StorePLocked as a
 8962 // pair to lock object allocations from Eden space when not using
 8963 // TLABs.
 8964 //
 8965 // There does not appear to be a Load{IL}Locked Ideal Node and the
 8966 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
 8967 // and to use StoreIConditional only for 32-bit and StoreLConditional
 8968 // only for 64-bit.
 8969 //
 8970 // We implement LoadPLocked and StorePLocked instructions using,
 8971 // respectively the AArch64 hw load-exclusive and store-conditional
 8972 // instructions. Whereas we must implement each of
 8973 // Store{IL}Conditional using a CAS which employs a pair of
 8974 // instructions comprising a load-exclusive followed by a
 8975 // store-conditional.
 8976 
 8977 
 8978 // Locked-load (linked load) of the current heap-top
 8979 // used when updating the eden heap top
 8980 // implemented using ldaxr on AArch64
 8981 
 8982 instruct loadPLocked(iRegPNoSp dst, indirect mem)
 8983 %{
 8984   match(Set dst (LoadPLocked mem));
 8985 
 8986   ins_cost(VOLATILE_REF_COST);
 8987 
 8988   format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
 8989 
 8990   ins_encode(aarch64_enc_ldaxr(dst, mem));
 8991 
 8992   ins_pipe(pipe_serial);
 8993 %}
 8994 
 8995 // Conditional-store of the updated heap-top.
 8996 // Used during allocation of the shared heap.
 8997 // Sets flag (EQ) on success.
 8998 // implemented using stlxr on AArch64.
 8999 
 9000 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
 9001 %{
 9002   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 9003 
 9004   ins_cost(VOLATILE_REF_COST);
 9005 
 9006  // TODO
 9007  // do we need to do a store-conditional release or can we just use a
 9008  // plain store-conditional?
 9009 
 9010   format %{
 9011     "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
 9012     "cmpw rscratch1, zr\t# EQ on successful write"
 9013   %}
 9014 
 9015   ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
 9016 
 9017   ins_pipe(pipe_serial);
 9018 %}
 9019 
 9020 
 9021 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
 9022 // when attempting to rebias a lock towards the current thread.  We
 9023 // must use the acquire form of cmpxchg in order to guarantee acquire
 9024 // semantics in this case.
 9025 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
 9026 %{
 9027   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 9028 
 9029   ins_cost(VOLATILE_REF_COST);
 9030 
 9031   format %{
 9032     "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9033     "cmpw rscratch1, zr\t# EQ on successful write"
 9034   %}
 9035 
 9036   ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
 9037 
 9038   ins_pipe(pipe_slow);
 9039 %}
 9040 
 9041 // storeIConditional also has acquire semantics, for no better reason
 9042 // than matching storeLConditional.  At the time of writing this
 9043 // comment storeIConditional was not used anywhere by AArch64.
 9044 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
 9045 %{
 9046   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 9047 
 9048   ins_cost(VOLATILE_REF_COST);
 9049 
 9050   format %{
 9051     "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
 9052     "cmpw rscratch1, zr\t# EQ on successful write"
 9053   %}
 9054 
 9055   ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
 9056 
 9057   ins_pipe(pipe_slow);
 9058 %}
 9059 
 9060 // standard CompareAndSwapX when we are using barriers
 9061 // these have higher priority than the rules selected by a predicate
 9062 
 9063 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 9064 // can't match them
 9065 
 9066 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9067 
 9068   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9069   ins_cost(2 * VOLATILE_REF_COST);
 9070 
 9071   effect(KILL cr);
 9072 
 9073   format %{
 9074     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9075     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9076   %}
 9077 
 9078   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 9079             aarch64_enc_cset_eq(res));
 9080 
 9081   ins_pipe(pipe_slow);
 9082 %}
 9083 
 9084 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9085 
 9086   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9087   ins_cost(2 * VOLATILE_REF_COST);
 9088 
 9089   effect(KILL cr);
 9090 
 9091   format %{
 9092     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9093     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9094   %}
 9095 
 9096   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 9097             aarch64_enc_cset_eq(res));
 9098 
 9099   ins_pipe(pipe_slow);
 9100 %}
 9101 
 9102 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9103 
 9104   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9105   ins_cost(2 * VOLATILE_REF_COST);
 9106 
 9107   effect(KILL cr);
 9108 
 9109  format %{
 9110     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9111     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9112  %}
 9113 
 9114  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9115             aarch64_enc_cset_eq(res));
 9116 
 9117   ins_pipe(pipe_slow);
 9118 %}
 9119 
 9120 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9121 
 9122   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9123   ins_cost(2 * VOLATILE_REF_COST);
 9124 
 9125   effect(KILL cr);
 9126 
 9127  format %{
 9128     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9129     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9130  %}
 9131 
 9132  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9133             aarch64_enc_cset_eq(res));
 9134 
 9135   ins_pipe(pipe_slow);
 9136 %}
 9137 
 9138 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9139 
 9140   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9141   predicate(n->as_LoadStore()->barrier_data() == 0);
 9142   ins_cost(2 * VOLATILE_REF_COST);
 9143 
 9144   effect(KILL cr);
 9145 
 9146  format %{
 9147     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9148     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9149  %}
 9150 
 9151  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 9152             aarch64_enc_cset_eq(res));
 9153 
 9154   ins_pipe(pipe_slow);
 9155 %}
 9156 
 9157 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9158 
 9159   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9160   ins_cost(2 * VOLATILE_REF_COST);
 9161 
 9162   effect(KILL cr);
 9163 
 9164  format %{
 9165     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9166     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9167  %}
 9168 
 9169  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 9170             aarch64_enc_cset_eq(res));
 9171 
 9172   ins_pipe(pipe_slow);
 9173 %}
 9174 
 9175 // alternative CompareAndSwapX when we are eliding barriers
 9176 
 9177 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9178 
 9179   predicate(needs_acquiring_load_exclusive(n));
 9180   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 9181   ins_cost(VOLATILE_REF_COST);
 9182 
 9183   effect(KILL cr);
 9184 
 9185   format %{
 9186     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9187     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9188   %}
 9189 
 9190   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 9191             aarch64_enc_cset_eq(res));
 9192 
 9193   ins_pipe(pipe_slow);
 9194 %}
 9195 
 9196 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9197 
 9198   predicate(needs_acquiring_load_exclusive(n));
 9199   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 9200   ins_cost(VOLATILE_REF_COST);
 9201 
 9202   effect(KILL cr);
 9203 
 9204   format %{
 9205     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9206     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9207   %}
 9208 
 9209   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 9210             aarch64_enc_cset_eq(res));
 9211 
 9212   ins_pipe(pipe_slow);
 9213 %}
 9214 
 9215 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 9216 
 9217   predicate(needs_acquiring_load_exclusive(n));
 9218   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9219   ins_cost(VOLATILE_REF_COST);
 9220 
 9221   effect(KILL cr);
 9222 
 9223  format %{
 9224     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9225     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9226  %}
 9227 
 9228  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9229             aarch64_enc_cset_eq(res));
 9230 
 9231   ins_pipe(pipe_slow);
 9232 %}
 9233 
 9234 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9235 
 9236   predicate(needs_acquiring_load_exclusive(n));
 9237   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9238   ins_cost(VOLATILE_REF_COST);
 9239 
 9240   effect(KILL cr);
 9241 
 9242  format %{
 9243     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9244     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9245  %}
 9246 
 9247  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9248             aarch64_enc_cset_eq(res));
 9249 
 9250   ins_pipe(pipe_slow);
 9251 %}
 9252 
 9253 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9254 
 9255   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9256   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9257   ins_cost(VOLATILE_REF_COST);
 9258 
 9259   effect(KILL cr);
 9260 
 9261  format %{
 9262     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9263     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9264  %}
 9265 
 9266  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9267             aarch64_enc_cset_eq(res));
 9268 
 9269   ins_pipe(pipe_slow);
 9270 %}
 9271 
 9272 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9273 
 9274   predicate(needs_acquiring_load_exclusive(n));
 9275   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9276   ins_cost(VOLATILE_REF_COST);
 9277 
 9278   effect(KILL cr);
 9279 
 9280  format %{
 9281     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9282     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9283  %}
 9284 
 9285  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9286             aarch64_enc_cset_eq(res));
 9287 
 9288   ins_pipe(pipe_slow);
 9289 %}
 9290 
 9291 
 9292 // ---------------------------------------------------------------------
 9293 
 9294 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9295 
 9296 // Sundry CAS operations.  Note that release is always true,
 9297 // regardless of the memory ordering of the CAS.  This is because we
 9298 // need the volatile case to be sequentially consistent but there is
 9299 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9300 // can't check the type of memory ordering here, so we always emit a
 9301 // STLXR.
 9302 
 9303 // This section is generated from aarch64_ad_cas.m4
 9304 
 9305 
 9306 
 9307 // This pattern is generated automatically from cas.m4.
 9308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9309 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9310 
 9311   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9312   ins_cost(2 * VOLATILE_REF_COST);
 9313   effect(TEMP_DEF res, KILL cr);
 9314   format %{
 9315     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9316   %}
 9317   ins_encode %{
 9318     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9319                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9320                /*weak*/ false, $res$$Register);
 9321     __ sxtbw($res$$Register, $res$$Register);
 9322   %}
 9323   ins_pipe(pipe_slow);
 9324 %}
 9325 
 9326 // This pattern is generated automatically from cas.m4.
 9327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9328 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9329 
 9330   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9331   ins_cost(2 * VOLATILE_REF_COST);
 9332   effect(TEMP_DEF res, KILL cr);
 9333   format %{
 9334     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9335   %}
 9336   ins_encode %{
 9337     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9338                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9339                /*weak*/ false, $res$$Register);
 9340     __ sxthw($res$$Register, $res$$Register);
 9341   %}
 9342   ins_pipe(pipe_slow);
 9343 %}
 9344 
 9345 // This pattern is generated automatically from cas.m4.
 9346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9347 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9348 
 9349   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9350   ins_cost(2 * VOLATILE_REF_COST);
 9351   effect(TEMP_DEF res, KILL cr);
 9352   format %{
 9353     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9354   %}
 9355   ins_encode %{
 9356     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9357                Assembler::word, /*acquire*/ false, /*release*/ true,
 9358                /*weak*/ false, $res$$Register);
 9359   %}
 9360   ins_pipe(pipe_slow);
 9361 %}
 9362 
 9363 // This pattern is generated automatically from cas.m4.
 9364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9365 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9366 
 9367   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9368   ins_cost(2 * VOLATILE_REF_COST);
 9369   effect(TEMP_DEF res, KILL cr);
 9370   format %{
 9371     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9372   %}
 9373   ins_encode %{
 9374     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9375                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9376                /*weak*/ false, $res$$Register);
 9377   %}
 9378   ins_pipe(pipe_slow);
 9379 %}
 9380 
 9381 // This pattern is generated automatically from cas.m4.
 9382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9383 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9384 
 9385   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9386   ins_cost(2 * VOLATILE_REF_COST);
 9387   effect(TEMP_DEF res, KILL cr);
 9388   format %{
 9389     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9390   %}
 9391   ins_encode %{
 9392     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9393                Assembler::word, /*acquire*/ false, /*release*/ true,
 9394                /*weak*/ false, $res$$Register);
 9395   %}
 9396   ins_pipe(pipe_slow);
 9397 %}
 9398 
 9399 // This pattern is generated automatically from cas.m4.
 9400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9401 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9402   predicate(n->as_LoadStore()->barrier_data() == 0);
 9403   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9404   ins_cost(2 * VOLATILE_REF_COST);
 9405   effect(TEMP_DEF res, KILL cr);
 9406   format %{
 9407     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9408   %}
 9409   ins_encode %{
 9410     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9411                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9412                /*weak*/ false, $res$$Register);
 9413   %}
 9414   ins_pipe(pipe_slow);
 9415 %}
 9416 
 9417 // This pattern is generated automatically from cas.m4.
 9418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9419 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9420   predicate(needs_acquiring_load_exclusive(n));
 9421   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9422   ins_cost(VOLATILE_REF_COST);
 9423   effect(TEMP_DEF res, KILL cr);
 9424   format %{
 9425     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9426   %}
 9427   ins_encode %{
 9428     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9429                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9430                /*weak*/ false, $res$$Register);
 9431     __ sxtbw($res$$Register, $res$$Register);
 9432   %}
 9433   ins_pipe(pipe_slow);
 9434 %}
 9435 
 9436 // This pattern is generated automatically from cas.m4.
 9437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9438 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9439   predicate(needs_acquiring_load_exclusive(n));
 9440   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9441   ins_cost(VOLATILE_REF_COST);
 9442   effect(TEMP_DEF res, KILL cr);
 9443   format %{
 9444     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9445   %}
 9446   ins_encode %{
 9447     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9448                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9449                /*weak*/ false, $res$$Register);
 9450     __ sxthw($res$$Register, $res$$Register);
 9451   %}
 9452   ins_pipe(pipe_slow);
 9453 %}
 9454 
 9455 // This pattern is generated automatically from cas.m4.
 9456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9457 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9458   predicate(needs_acquiring_load_exclusive(n));
 9459   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9460   ins_cost(VOLATILE_REF_COST);
 9461   effect(TEMP_DEF res, KILL cr);
 9462   format %{
 9463     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9464   %}
 9465   ins_encode %{
 9466     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9467                Assembler::word, /*acquire*/ true, /*release*/ true,
 9468                /*weak*/ false, $res$$Register);
 9469   %}
 9470   ins_pipe(pipe_slow);
 9471 %}
 9472 
 9473 // This pattern is generated automatically from cas.m4.
 9474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9475 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9476   predicate(needs_acquiring_load_exclusive(n));
 9477   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9478   ins_cost(VOLATILE_REF_COST);
 9479   effect(TEMP_DEF res, KILL cr);
 9480   format %{
 9481     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9482   %}
 9483   ins_encode %{
 9484     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9485                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9486                /*weak*/ false, $res$$Register);
 9487   %}
 9488   ins_pipe(pipe_slow);
 9489 %}
 9490 
 9491 // This pattern is generated automatically from cas.m4.
 9492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9493 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9494   predicate(needs_acquiring_load_exclusive(n));
 9495   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9496   ins_cost(VOLATILE_REF_COST);
 9497   effect(TEMP_DEF res, KILL cr);
 9498   format %{
 9499     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9500   %}
 9501   ins_encode %{
 9502     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9503                Assembler::word, /*acquire*/ true, /*release*/ true,
 9504                /*weak*/ false, $res$$Register);
 9505   %}
 9506   ins_pipe(pipe_slow);
 9507 %}
 9508 
 9509 // This pattern is generated automatically from cas.m4.
 9510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9511 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9512   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9513   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9514   ins_cost(VOLATILE_REF_COST);
 9515   effect(TEMP_DEF res, KILL cr);
 9516   format %{
 9517     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9518   %}
 9519   ins_encode %{
 9520     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9521                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9522                /*weak*/ false, $res$$Register);
 9523   %}
 9524   ins_pipe(pipe_slow);
 9525 %}
 9526 
 9527 // This pattern is generated automatically from cas.m4.
 9528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9529 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9530 
 9531   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9532   ins_cost(2 * VOLATILE_REF_COST);
 9533   effect(KILL cr);
 9534   format %{
 9535     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9536     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9537   %}
 9538   ins_encode %{
 9539     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9540                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9541                /*weak*/ true, noreg);
 9542     __ csetw($res$$Register, Assembler::EQ);
 9543   %}
 9544   ins_pipe(pipe_slow);
 9545 %}
 9546 
 9547 // This pattern is generated automatically from cas.m4.
 9548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9549 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9550 
 9551   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9552   ins_cost(2 * VOLATILE_REF_COST);
 9553   effect(KILL cr);
 9554   format %{
 9555     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9556     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9557   %}
 9558   ins_encode %{
 9559     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9560                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9561                /*weak*/ true, noreg);
 9562     __ csetw($res$$Register, Assembler::EQ);
 9563   %}
 9564   ins_pipe(pipe_slow);
 9565 %}
 9566 
 9567 // This pattern is generated automatically from cas.m4.
 9568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9569 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9570 
 9571   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9572   ins_cost(2 * VOLATILE_REF_COST);
 9573   effect(KILL cr);
 9574   format %{
 9575     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9576     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9577   %}
 9578   ins_encode %{
 9579     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9580                Assembler::word, /*acquire*/ false, /*release*/ true,
 9581                /*weak*/ true, noreg);
 9582     __ csetw($res$$Register, Assembler::EQ);
 9583   %}
 9584   ins_pipe(pipe_slow);
 9585 %}
 9586 
 9587 // This pattern is generated automatically from cas.m4.
 9588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9589 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9590 
 9591   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9592   ins_cost(2 * VOLATILE_REF_COST);
 9593   effect(KILL cr);
 9594   format %{
 9595     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9596     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9597   %}
 9598   ins_encode %{
 9599     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9600                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9601                /*weak*/ true, noreg);
 9602     __ csetw($res$$Register, Assembler::EQ);
 9603   %}
 9604   ins_pipe(pipe_slow);
 9605 %}
 9606 
 9607 // This pattern is generated automatically from cas.m4.
 9608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9609 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9610 
 9611   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9612   ins_cost(2 * VOLATILE_REF_COST);
 9613   effect(KILL cr);
 9614   format %{
 9615     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9616     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9617   %}
 9618   ins_encode %{
 9619     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9620                Assembler::word, /*acquire*/ false, /*release*/ true,
 9621                /*weak*/ true, noreg);
 9622     __ csetw($res$$Register, Assembler::EQ);
 9623   %}
 9624   ins_pipe(pipe_slow);
 9625 %}
 9626 
 9627 // This pattern is generated automatically from cas.m4.
 9628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9629 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9630   predicate(n->as_LoadStore()->barrier_data() == 0);
 9631   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9632   ins_cost(2 * VOLATILE_REF_COST);
 9633   effect(KILL cr);
 9634   format %{
 9635     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9636     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9637   %}
 9638   ins_encode %{
 9639     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9640                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9641                /*weak*/ true, noreg);
 9642     __ csetw($res$$Register, Assembler::EQ);
 9643   %}
 9644   ins_pipe(pipe_slow);
 9645 %}
 9646 
 9647 // This pattern is generated automatically from cas.m4.
 9648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9649 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9650   predicate(needs_acquiring_load_exclusive(n));
 9651   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9652   ins_cost(VOLATILE_REF_COST);
 9653   effect(KILL cr);
 9654   format %{
 9655     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9656     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9657   %}
 9658   ins_encode %{
 9659     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9660                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9661                /*weak*/ true, noreg);
 9662     __ csetw($res$$Register, Assembler::EQ);
 9663   %}
 9664   ins_pipe(pipe_slow);
 9665 %}
 9666 
 9667 // This pattern is generated automatically from cas.m4.
 9668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9669 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9670   predicate(needs_acquiring_load_exclusive(n));
 9671   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9672   ins_cost(VOLATILE_REF_COST);
 9673   effect(KILL cr);
 9674   format %{
 9675     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9676     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9677   %}
 9678   ins_encode %{
 9679     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9680                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9681                /*weak*/ true, noreg);
 9682     __ csetw($res$$Register, Assembler::EQ);
 9683   %}
 9684   ins_pipe(pipe_slow);
 9685 %}
 9686 
 9687 // This pattern is generated automatically from cas.m4.
 9688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9689 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9690   predicate(needs_acquiring_load_exclusive(n));
 9691   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9692   ins_cost(VOLATILE_REF_COST);
 9693   effect(KILL cr);
 9694   format %{
 9695     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9696     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9697   %}
 9698   ins_encode %{
 9699     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9700                Assembler::word, /*acquire*/ true, /*release*/ true,
 9701                /*weak*/ true, noreg);
 9702     __ csetw($res$$Register, Assembler::EQ);
 9703   %}
 9704   ins_pipe(pipe_slow);
 9705 %}
 9706 
 9707 // This pattern is generated automatically from cas.m4.
 9708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9709 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9710   predicate(needs_acquiring_load_exclusive(n));
 9711   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9712   ins_cost(VOLATILE_REF_COST);
 9713   effect(KILL cr);
 9714   format %{
 9715     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9716     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9717   %}
 9718   ins_encode %{
 9719     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9720                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9721                /*weak*/ true, noreg);
 9722     __ csetw($res$$Register, Assembler::EQ);
 9723   %}
 9724   ins_pipe(pipe_slow);
 9725 %}
 9726 
 9727 // This pattern is generated automatically from cas.m4.
 9728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9729 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9730   predicate(needs_acquiring_load_exclusive(n));
 9731   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9732   ins_cost(VOLATILE_REF_COST);
 9733   effect(KILL cr);
 9734   format %{
 9735     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9736     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9737   %}
 9738   ins_encode %{
 9739     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9740                Assembler::word, /*acquire*/ true, /*release*/ true,
 9741                /*weak*/ true, noreg);
 9742     __ csetw($res$$Register, Assembler::EQ);
 9743   %}
 9744   ins_pipe(pipe_slow);
 9745 %}
 9746 
 9747 // This pattern is generated automatically from cas.m4.
 9748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9749 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9750   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9751   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9752   ins_cost(VOLATILE_REF_COST);
 9753   effect(KILL cr);
 9754   format %{
 9755     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9756     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9757   %}
 9758   ins_encode %{
 9759     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9760                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9761                /*weak*/ true, noreg);
 9762     __ csetw($res$$Register, Assembler::EQ);
 9763   %}
 9764   ins_pipe(pipe_slow);
 9765 %}
 9766 
 9767 // END This section of the file is automatically generated. Do not edit --------------
 9768 // ---------------------------------------------------------------------
 9769 
 9770 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9771   match(Set prev (GetAndSetI mem newv));
 9772   ins_cost(2 * VOLATILE_REF_COST);
 9773   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9774   ins_encode %{
 9775     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9776   %}
 9777   ins_pipe(pipe_serial);
 9778 %}
 9779 
 9780 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9781   match(Set prev (GetAndSetL mem newv));
 9782   ins_cost(2 * VOLATILE_REF_COST);
 9783   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9784   ins_encode %{
 9785     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9786   %}
 9787   ins_pipe(pipe_serial);
 9788 %}
 9789 
 9790 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9791   match(Set prev (GetAndSetN mem newv));
 9792   ins_cost(2 * VOLATILE_REF_COST);
 9793   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9794   ins_encode %{
 9795     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9796   %}
 9797   ins_pipe(pipe_serial);
 9798 %}
 9799 
 9800 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9801   predicate(n->as_LoadStore()->barrier_data() == 0);
 9802   match(Set prev (GetAndSetP mem newv));
 9803   ins_cost(2 * VOLATILE_REF_COST);
 9804   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9805   ins_encode %{
 9806     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9807   %}
 9808   ins_pipe(pipe_serial);
 9809 %}
 9810 
 9811 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9812   predicate(needs_acquiring_load_exclusive(n));
 9813   match(Set prev (GetAndSetI mem newv));
 9814   ins_cost(VOLATILE_REF_COST);
 9815   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9816   ins_encode %{
 9817     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9818   %}
 9819   ins_pipe(pipe_serial);
 9820 %}
 9821 
 9822 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9823   predicate(needs_acquiring_load_exclusive(n));
 9824   match(Set prev (GetAndSetL mem newv));
 9825   ins_cost(VOLATILE_REF_COST);
 9826   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9827   ins_encode %{
 9828     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9829   %}
 9830   ins_pipe(pipe_serial);
 9831 %}
 9832 
 9833 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9834   predicate(needs_acquiring_load_exclusive(n));
 9835   match(Set prev (GetAndSetN mem newv));
 9836   ins_cost(VOLATILE_REF_COST);
 9837   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9838   ins_encode %{
 9839     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9840   %}
 9841   ins_pipe(pipe_serial);
 9842 %}
 9843 
 9844 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9845   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9846   match(Set prev (GetAndSetP mem newv));
 9847   ins_cost(VOLATILE_REF_COST);
 9848   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9849   ins_encode %{
 9850     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9851   %}
 9852   ins_pipe(pipe_serial);
 9853 %}
 9854 
 9855 
 9856 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9857   match(Set newval (GetAndAddL mem incr));
 9858   ins_cost(2 * VOLATILE_REF_COST + 1);
 9859   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9860   ins_encode %{
 9861     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9862   %}
 9863   ins_pipe(pipe_serial);
 9864 %}
 9865 
 9866 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9867   predicate(n->as_LoadStore()->result_not_used());
 9868   match(Set dummy (GetAndAddL mem incr));
 9869   ins_cost(2 * VOLATILE_REF_COST);
 9870   format %{ "get_and_addL [$mem], $incr" %}
 9871   ins_encode %{
 9872     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9873   %}
 9874   ins_pipe(pipe_serial);
 9875 %}
 9876 
 9877 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9878   match(Set newval (GetAndAddL mem incr));
 9879   ins_cost(2 * VOLATILE_REF_COST + 1);
 9880   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9881   ins_encode %{
 9882     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9883   %}
 9884   ins_pipe(pipe_serial);
 9885 %}
 9886 
 9887 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9888   predicate(n->as_LoadStore()->result_not_used());
 9889   match(Set dummy (GetAndAddL mem incr));
 9890   ins_cost(2 * VOLATILE_REF_COST);
 9891   format %{ "get_and_addL [$mem], $incr" %}
 9892   ins_encode %{
 9893     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9894   %}
 9895   ins_pipe(pipe_serial);
 9896 %}
 9897 
 9898 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9899   match(Set newval (GetAndAddI mem incr));
 9900   ins_cost(2 * VOLATILE_REF_COST + 1);
 9901   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9902   ins_encode %{
 9903     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9904   %}
 9905   ins_pipe(pipe_serial);
 9906 %}
 9907 
 9908 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9909   predicate(n->as_LoadStore()->result_not_used());
 9910   match(Set dummy (GetAndAddI mem incr));
 9911   ins_cost(2 * VOLATILE_REF_COST);
 9912   format %{ "get_and_addI [$mem], $incr" %}
 9913   ins_encode %{
 9914     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9915   %}
 9916   ins_pipe(pipe_serial);
 9917 %}
 9918 
 9919 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9920   match(Set newval (GetAndAddI mem incr));
 9921   ins_cost(2 * VOLATILE_REF_COST + 1);
 9922   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9923   ins_encode %{
 9924     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9925   %}
 9926   ins_pipe(pipe_serial);
 9927 %}
 9928 
 9929 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9930   predicate(n->as_LoadStore()->result_not_used());
 9931   match(Set dummy (GetAndAddI mem incr));
 9932   ins_cost(2 * VOLATILE_REF_COST);
 9933   format %{ "get_and_addI [$mem], $incr" %}
 9934   ins_encode %{
 9935     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9936   %}
 9937   ins_pipe(pipe_serial);
 9938 %}
 9939 
 9940 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9941   predicate(needs_acquiring_load_exclusive(n));
 9942   match(Set newval (GetAndAddL mem incr));
 9943   ins_cost(VOLATILE_REF_COST + 1);
 9944   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9945   ins_encode %{
 9946     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9947   %}
 9948   ins_pipe(pipe_serial);
 9949 %}
 9950 
 9951 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9952   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9953   match(Set dummy (GetAndAddL mem incr));
 9954   ins_cost(VOLATILE_REF_COST);
 9955   format %{ "get_and_addL_acq [$mem], $incr" %}
 9956   ins_encode %{
 9957     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9958   %}
 9959   ins_pipe(pipe_serial);
 9960 %}
 9961 
 9962 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9963   predicate(needs_acquiring_load_exclusive(n));
 9964   match(Set newval (GetAndAddL mem incr));
 9965   ins_cost(VOLATILE_REF_COST + 1);
 9966   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9967   ins_encode %{
 9968     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9969   %}
 9970   ins_pipe(pipe_serial);
 9971 %}
 9972 
 9973 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9974   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9975   match(Set dummy (GetAndAddL mem incr));
 9976   ins_cost(VOLATILE_REF_COST);
 9977   format %{ "get_and_addL_acq [$mem], $incr" %}
 9978   ins_encode %{
 9979     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9980   %}
 9981   ins_pipe(pipe_serial);
 9982 %}
 9983 
 9984 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9985   predicate(needs_acquiring_load_exclusive(n));
 9986   match(Set newval (GetAndAddI mem incr));
 9987   ins_cost(VOLATILE_REF_COST + 1);
 9988   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9989   ins_encode %{
 9990     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9991   %}
 9992   ins_pipe(pipe_serial);
 9993 %}
 9994 
 9995 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9996   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9997   match(Set dummy (GetAndAddI mem incr));
 9998   ins_cost(VOLATILE_REF_COST);
 9999   format %{ "get_and_addI_acq [$mem], $incr" %}
10000   ins_encode %{
10001     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10002   %}
10003   ins_pipe(pipe_serial);
10004 %}
10005 
10006 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10007   predicate(needs_acquiring_load_exclusive(n));
10008   match(Set newval (GetAndAddI mem incr));
10009   ins_cost(VOLATILE_REF_COST + 1);
10010   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10011   ins_encode %{
10012     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10013   %}
10014   ins_pipe(pipe_serial);
10015 %}
10016 
10017 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10018   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10019   match(Set dummy (GetAndAddI mem incr));
10020   ins_cost(VOLATILE_REF_COST);
10021   format %{ "get_and_addI_acq [$mem], $incr" %}
10022   ins_encode %{
10023     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10024   %}
10025   ins_pipe(pipe_serial);
10026 %}
10027 
10028 // Manifest a CmpL result in an integer register.
10029 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10030 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10031 %{
10032   match(Set dst (CmpL3 src1 src2));
10033   effect(KILL flags);
10034 
10035   ins_cost(INSN_COST * 6);
10036   format %{
10037       "cmp $src1, $src2"
10038       "csetw $dst, ne"
10039       "cnegw $dst, lt"
10040   %}
10041   // format %{ "CmpL3 $dst, $src1, $src2" %}
10042   ins_encode %{
10043     __ cmp($src1$$Register, $src2$$Register);
10044     __ csetw($dst$$Register, Assembler::NE);
10045     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10046   %}
10047 
10048   ins_pipe(pipe_class_default);
10049 %}
10050 
10051 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10052 %{
10053   match(Set dst (CmpL3 src1 src2));
10054   effect(KILL flags);
10055 
10056   ins_cost(INSN_COST * 6);
10057   format %{
10058       "cmp $src1, $src2"
10059       "csetw $dst, ne"
10060       "cnegw $dst, lt"
10061   %}
10062   ins_encode %{
10063     int32_t con = (int32_t)$src2$$constant;
10064      if (con < 0) {
10065       __ adds(zr, $src1$$Register, -con);
10066     } else {
10067       __ subs(zr, $src1$$Register, con);
10068     }
10069     __ csetw($dst$$Register, Assembler::NE);
10070     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10071   %}
10072 
10073   ins_pipe(pipe_class_default);
10074 %}
10075 
10076 // ============================================================================
10077 // Conditional Move Instructions
10078 
10079 // n.b. we have identical rules for both a signed compare op (cmpOp)
10080 // and an unsigned compare op (cmpOpU). it would be nice if we could
10081 // define an op class which merged both inputs and use it to type the
10082 // argument to a single rule. unfortunatelyt his fails because the
10083 // opclass does not live up to the COND_INTER interface of its
10084 // component operands. When the generic code tries to negate the
10085 // operand it ends up running the generci Machoper::negate method
10086 // which throws a ShouldNotHappen. So, we have to provide two flavours
10087 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10088 
10089 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10090   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10091 
10092   ins_cost(INSN_COST * 2);
10093   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
10094 
10095   ins_encode %{
10096     __ cselw(as_Register($dst$$reg),
10097              as_Register($src2$$reg),
10098              as_Register($src1$$reg),
10099              (Assembler::Condition)$cmp$$cmpcode);
10100   %}
10101 
10102   ins_pipe(icond_reg_reg);
10103 %}
10104 
10105 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10106   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10107 
10108   ins_cost(INSN_COST * 2);
10109   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
10110 
10111   ins_encode %{
10112     __ cselw(as_Register($dst$$reg),
10113              as_Register($src2$$reg),
10114              as_Register($src1$$reg),
10115              (Assembler::Condition)$cmp$$cmpcode);
10116   %}
10117 
10118   ins_pipe(icond_reg_reg);
10119 %}
10120 
10121 // special cases where one arg is zero
10122 
10123 // n.b. this is selected in preference to the rule above because it
10124 // avoids loading constant 0 into a source register
10125 
10126 // TODO
10127 // we ought only to be able to cull one of these variants as the ideal
10128 // transforms ought always to order the zero consistently (to left/right?)
10129 
10130 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10131   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10132 
10133   ins_cost(INSN_COST * 2);
10134   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
10135 
10136   ins_encode %{
10137     __ cselw(as_Register($dst$$reg),
10138              as_Register($src$$reg),
10139              zr,
10140              (Assembler::Condition)$cmp$$cmpcode);
10141   %}
10142 
10143   ins_pipe(icond_reg);
10144 %}
10145 
10146 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10147   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10148 
10149   ins_cost(INSN_COST * 2);
10150   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10151 
10152   ins_encode %{
10153     __ cselw(as_Register($dst$$reg),
10154              as_Register($src$$reg),
10155              zr,
10156              (Assembler::Condition)$cmp$$cmpcode);
10157   %}
10158 
10159   ins_pipe(icond_reg);
10160 %}
10161 
10162 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10163   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10164 
10165   ins_cost(INSN_COST * 2);
10166   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10167 
10168   ins_encode %{
10169     __ cselw(as_Register($dst$$reg),
10170              zr,
10171              as_Register($src$$reg),
10172              (Assembler::Condition)$cmp$$cmpcode);
10173   %}
10174 
10175   ins_pipe(icond_reg);
10176 %}
10177 
10178 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10179   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10180 
10181   ins_cost(INSN_COST * 2);
10182   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10183 
10184   ins_encode %{
10185     __ cselw(as_Register($dst$$reg),
10186              zr,
10187              as_Register($src$$reg),
10188              (Assembler::Condition)$cmp$$cmpcode);
10189   %}
10190 
10191   ins_pipe(icond_reg);
10192 %}
10193 
10194 // special case for creating a boolean 0 or 1
10195 
10196 // n.b. this is selected in preference to the rule above because it
10197 // avoids loading constants 0 and 1 into a source register
10198 
10199 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10200   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10201 
10202   ins_cost(INSN_COST * 2);
10203   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10204 
10205   ins_encode %{
10206     // equivalently
10207     // cset(as_Register($dst$$reg),
10208     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10209     __ csincw(as_Register($dst$$reg),
10210              zr,
10211              zr,
10212              (Assembler::Condition)$cmp$$cmpcode);
10213   %}
10214 
10215   ins_pipe(icond_none);
10216 %}
10217 
10218 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10219   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10220 
10221   ins_cost(INSN_COST * 2);
10222   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10223 
10224   ins_encode %{
10225     // equivalently
10226     // cset(as_Register($dst$$reg),
10227     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10228     __ csincw(as_Register($dst$$reg),
10229              zr,
10230              zr,
10231              (Assembler::Condition)$cmp$$cmpcode);
10232   %}
10233 
10234   ins_pipe(icond_none);
10235 %}
10236 
10237 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10238   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10239 
10240   ins_cost(INSN_COST * 2);
10241   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10242 
10243   ins_encode %{
10244     __ csel(as_Register($dst$$reg),
10245             as_Register($src2$$reg),
10246             as_Register($src1$$reg),
10247             (Assembler::Condition)$cmp$$cmpcode);
10248   %}
10249 
10250   ins_pipe(icond_reg_reg);
10251 %}
10252 
10253 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10254   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10255 
10256   ins_cost(INSN_COST * 2);
10257   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10258 
10259   ins_encode %{
10260     __ csel(as_Register($dst$$reg),
10261             as_Register($src2$$reg),
10262             as_Register($src1$$reg),
10263             (Assembler::Condition)$cmp$$cmpcode);
10264   %}
10265 
10266   ins_pipe(icond_reg_reg);
10267 %}
10268 
10269 // special cases where one arg is zero
10270 
10271 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10272   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10273 
10274   ins_cost(INSN_COST * 2);
10275   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10276 
10277   ins_encode %{
10278     __ csel(as_Register($dst$$reg),
10279             zr,
10280             as_Register($src$$reg),
10281             (Assembler::Condition)$cmp$$cmpcode);
10282   %}
10283 
10284   ins_pipe(icond_reg);
10285 %}
10286 
10287 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10288   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10289 
10290   ins_cost(INSN_COST * 2);
10291   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10292 
10293   ins_encode %{
10294     __ csel(as_Register($dst$$reg),
10295             zr,
10296             as_Register($src$$reg),
10297             (Assembler::Condition)$cmp$$cmpcode);
10298   %}
10299 
10300   ins_pipe(icond_reg);
10301 %}
10302 
10303 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10304   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10305 
10306   ins_cost(INSN_COST * 2);
10307   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10308 
10309   ins_encode %{
10310     __ csel(as_Register($dst$$reg),
10311             as_Register($src$$reg),
10312             zr,
10313             (Assembler::Condition)$cmp$$cmpcode);
10314   %}
10315 
10316   ins_pipe(icond_reg);
10317 %}
10318 
10319 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10320   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10321 
10322   ins_cost(INSN_COST * 2);
10323   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10324 
10325   ins_encode %{
10326     __ csel(as_Register($dst$$reg),
10327             as_Register($src$$reg),
10328             zr,
10329             (Assembler::Condition)$cmp$$cmpcode);
10330   %}
10331 
10332   ins_pipe(icond_reg);
10333 %}
10334 
10335 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10336   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10337 
10338   ins_cost(INSN_COST * 2);
10339   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10340 
10341   ins_encode %{
10342     __ csel(as_Register($dst$$reg),
10343             as_Register($src2$$reg),
10344             as_Register($src1$$reg),
10345             (Assembler::Condition)$cmp$$cmpcode);
10346   %}
10347 
10348   ins_pipe(icond_reg_reg);
10349 %}
10350 
10351 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10352   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10353 
10354   ins_cost(INSN_COST * 2);
10355   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10356 
10357   ins_encode %{
10358     __ csel(as_Register($dst$$reg),
10359             as_Register($src2$$reg),
10360             as_Register($src1$$reg),
10361             (Assembler::Condition)$cmp$$cmpcode);
10362   %}
10363 
10364   ins_pipe(icond_reg_reg);
10365 %}
10366 
10367 // special cases where one arg is zero
10368 
10369 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10370   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10371 
10372   ins_cost(INSN_COST * 2);
10373   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10374 
10375   ins_encode %{
10376     __ csel(as_Register($dst$$reg),
10377             zr,
10378             as_Register($src$$reg),
10379             (Assembler::Condition)$cmp$$cmpcode);
10380   %}
10381 
10382   ins_pipe(icond_reg);
10383 %}
10384 
10385 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10386   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10387 
10388   ins_cost(INSN_COST * 2);
10389   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10390 
10391   ins_encode %{
10392     __ csel(as_Register($dst$$reg),
10393             zr,
10394             as_Register($src$$reg),
10395             (Assembler::Condition)$cmp$$cmpcode);
10396   %}
10397 
10398   ins_pipe(icond_reg);
10399 %}
10400 
10401 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10402   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10403 
10404   ins_cost(INSN_COST * 2);
10405   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10406 
10407   ins_encode %{
10408     __ csel(as_Register($dst$$reg),
10409             as_Register($src$$reg),
10410             zr,
10411             (Assembler::Condition)$cmp$$cmpcode);
10412   %}
10413 
10414   ins_pipe(icond_reg);
10415 %}
10416 
10417 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10418   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10419 
10420   ins_cost(INSN_COST * 2);
10421   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10422 
10423   ins_encode %{
10424     __ csel(as_Register($dst$$reg),
10425             as_Register($src$$reg),
10426             zr,
10427             (Assembler::Condition)$cmp$$cmpcode);
10428   %}
10429 
10430   ins_pipe(icond_reg);
10431 %}
10432 
10433 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10434   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10435 
10436   ins_cost(INSN_COST * 2);
10437   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10438 
10439   ins_encode %{
10440     __ cselw(as_Register($dst$$reg),
10441              as_Register($src2$$reg),
10442              as_Register($src1$$reg),
10443              (Assembler::Condition)$cmp$$cmpcode);
10444   %}
10445 
10446   ins_pipe(icond_reg_reg);
10447 %}
10448 
10449 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10450   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10451 
10452   ins_cost(INSN_COST * 2);
10453   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10454 
10455   ins_encode %{
10456     __ cselw(as_Register($dst$$reg),
10457              as_Register($src2$$reg),
10458              as_Register($src1$$reg),
10459              (Assembler::Condition)$cmp$$cmpcode);
10460   %}
10461 
10462   ins_pipe(icond_reg_reg);
10463 %}
10464 
10465 // special cases where one arg is zero
10466 
10467 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10468   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10469 
10470   ins_cost(INSN_COST * 2);
10471   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10472 
10473   ins_encode %{
10474     __ cselw(as_Register($dst$$reg),
10475              zr,
10476              as_Register($src$$reg),
10477              (Assembler::Condition)$cmp$$cmpcode);
10478   %}
10479 
10480   ins_pipe(icond_reg);
10481 %}
10482 
10483 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10484   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10485 
10486   ins_cost(INSN_COST * 2);
10487   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10488 
10489   ins_encode %{
10490     __ cselw(as_Register($dst$$reg),
10491              zr,
10492              as_Register($src$$reg),
10493              (Assembler::Condition)$cmp$$cmpcode);
10494   %}
10495 
10496   ins_pipe(icond_reg);
10497 %}
10498 
10499 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10500   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10501 
10502   ins_cost(INSN_COST * 2);
10503   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10504 
10505   ins_encode %{
10506     __ cselw(as_Register($dst$$reg),
10507              as_Register($src$$reg),
10508              zr,
10509              (Assembler::Condition)$cmp$$cmpcode);
10510   %}
10511 
10512   ins_pipe(icond_reg);
10513 %}
10514 
10515 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10516   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10517 
10518   ins_cost(INSN_COST * 2);
10519   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10520 
10521   ins_encode %{
10522     __ cselw(as_Register($dst$$reg),
10523              as_Register($src$$reg),
10524              zr,
10525              (Assembler::Condition)$cmp$$cmpcode);
10526   %}
10527 
10528   ins_pipe(icond_reg);
10529 %}
10530 
10531 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10532 %{
10533   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10534 
10535   ins_cost(INSN_COST * 3);
10536 
10537   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10538   ins_encode %{
10539     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10540     __ fcsels(as_FloatRegister($dst$$reg),
10541               as_FloatRegister($src2$$reg),
10542               as_FloatRegister($src1$$reg),
10543               cond);
10544   %}
10545 
10546   ins_pipe(fp_cond_reg_reg_s);
10547 %}
10548 
10549 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10550 %{
10551   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10552 
10553   ins_cost(INSN_COST * 3);
10554 
10555   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10556   ins_encode %{
10557     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10558     __ fcsels(as_FloatRegister($dst$$reg),
10559               as_FloatRegister($src2$$reg),
10560               as_FloatRegister($src1$$reg),
10561               cond);
10562   %}
10563 
10564   ins_pipe(fp_cond_reg_reg_s);
10565 %}
10566 
10567 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10568 %{
10569   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10570 
10571   ins_cost(INSN_COST * 3);
10572 
10573   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10574   ins_encode %{
10575     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10576     __ fcseld(as_FloatRegister($dst$$reg),
10577               as_FloatRegister($src2$$reg),
10578               as_FloatRegister($src1$$reg),
10579               cond);
10580   %}
10581 
10582   ins_pipe(fp_cond_reg_reg_d);
10583 %}
10584 
10585 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10586 %{
10587   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10588 
10589   ins_cost(INSN_COST * 3);
10590 
10591   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10592   ins_encode %{
10593     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10594     __ fcseld(as_FloatRegister($dst$$reg),
10595               as_FloatRegister($src2$$reg),
10596               as_FloatRegister($src1$$reg),
10597               cond);
10598   %}
10599 
10600   ins_pipe(fp_cond_reg_reg_d);
10601 %}
10602 
10603 // ============================================================================
10604 // Arithmetic Instructions
10605 //
10606 
10607 // Integer Addition
10608 
10609 // TODO
10610 // these currently employ operations which do not set CR and hence are
10611 // not flagged as killing CR but we would like to isolate the cases
10612 // where we want to set flags from those where we don't. need to work
10613 // out how to do that.
10614 
10615 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10616   match(Set dst (AddI src1 src2));
10617 
10618   ins_cost(INSN_COST);
10619   format %{ "addw  $dst, $src1, $src2" %}
10620 
10621   ins_encode %{
10622     __ addw(as_Register($dst$$reg),
10623             as_Register($src1$$reg),
10624             as_Register($src2$$reg));
10625   %}
10626 
10627   ins_pipe(ialu_reg_reg);
10628 %}
10629 
10630 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10631   match(Set dst (AddI src1 src2));
10632 
10633   ins_cost(INSN_COST);
10634   format %{ "addw $dst, $src1, $src2" %}
10635 
10636   // use opcode to indicate that this is an add not a sub
10637   opcode(0x0);
10638 
10639   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10640 
10641   ins_pipe(ialu_reg_imm);
10642 %}
10643 
10644 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10645   match(Set dst (AddI (ConvL2I src1) src2));
10646 
10647   ins_cost(INSN_COST);
10648   format %{ "addw $dst, $src1, $src2" %}
10649 
10650   // use opcode to indicate that this is an add not a sub
10651   opcode(0x0);
10652 
10653   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10654 
10655   ins_pipe(ialu_reg_imm);
10656 %}
10657 
10658 // Pointer Addition
10659 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10660   match(Set dst (AddP src1 src2));
10661 
10662   ins_cost(INSN_COST);
10663   format %{ "add $dst, $src1, $src2\t# ptr" %}
10664 
10665   ins_encode %{
10666     __ add(as_Register($dst$$reg),
10667            as_Register($src1$$reg),
10668            as_Register($src2$$reg));
10669   %}
10670 
10671   ins_pipe(ialu_reg_reg);
10672 %}
10673 
10674 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10675   match(Set dst (AddP src1 (ConvI2L src2)));
10676 
10677   ins_cost(1.9 * INSN_COST);
10678   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10679 
10680   ins_encode %{
10681     __ add(as_Register($dst$$reg),
10682            as_Register($src1$$reg),
10683            as_Register($src2$$reg), ext::sxtw);
10684   %}
10685 
10686   ins_pipe(ialu_reg_reg);
10687 %}
10688 
10689 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10690   match(Set dst (AddP src1 (LShiftL src2 scale)));
10691 
10692   ins_cost(1.9 * INSN_COST);
10693   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10694 
10695   ins_encode %{
10696     __ lea(as_Register($dst$$reg),
10697            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10698                    Address::lsl($scale$$constant)));
10699   %}
10700 
10701   ins_pipe(ialu_reg_reg_shift);
10702 %}
10703 
10704 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10705   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10706 
10707   ins_cost(1.9 * INSN_COST);
10708   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10709 
10710   ins_encode %{
10711     __ lea(as_Register($dst$$reg),
10712            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10713                    Address::sxtw($scale$$constant)));
10714   %}
10715 
10716   ins_pipe(ialu_reg_reg_shift);
10717 %}
10718 
10719 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10720   match(Set dst (LShiftL (ConvI2L src) scale));
10721 
10722   ins_cost(INSN_COST);
10723   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10724 
10725   ins_encode %{
10726     __ sbfiz(as_Register($dst$$reg),
10727           as_Register($src$$reg),
10728           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10729   %}
10730 
10731   ins_pipe(ialu_reg_shift);
10732 %}
10733 
10734 // Pointer Immediate Addition
10735 // n.b. this needs to be more expensive than using an indirect memory
10736 // operand
10737 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10738   match(Set dst (AddP src1 src2));
10739 
10740   ins_cost(INSN_COST);
10741   format %{ "add $dst, $src1, $src2\t# ptr" %}
10742 
10743   // use opcode to indicate that this is an add not a sub
10744   opcode(0x0);
10745 
10746   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10747 
10748   ins_pipe(ialu_reg_imm);
10749 %}
10750 
10751 // Long Addition
10752 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10753 
10754   match(Set dst (AddL src1 src2));
10755 
10756   ins_cost(INSN_COST);
10757   format %{ "add  $dst, $src1, $src2" %}
10758 
10759   ins_encode %{
10760     __ add(as_Register($dst$$reg),
10761            as_Register($src1$$reg),
10762            as_Register($src2$$reg));
10763   %}
10764 
10765   ins_pipe(ialu_reg_reg);
10766 %}
10767 
10768 // No constant pool entries requiredLong Immediate Addition.
10769 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10770   match(Set dst (AddL src1 src2));
10771 
10772   ins_cost(INSN_COST);
10773   format %{ "add $dst, $src1, $src2" %}
10774 
10775   // use opcode to indicate that this is an add not a sub
10776   opcode(0x0);
10777 
10778   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10779 
10780   ins_pipe(ialu_reg_imm);
10781 %}
10782 
10783 // Integer Subtraction
10784 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10785   match(Set dst (SubI src1 src2));
10786 
10787   ins_cost(INSN_COST);
10788   format %{ "subw  $dst, $src1, $src2" %}
10789 
10790   ins_encode %{
10791     __ subw(as_Register($dst$$reg),
10792             as_Register($src1$$reg),
10793             as_Register($src2$$reg));
10794   %}
10795 
10796   ins_pipe(ialu_reg_reg);
10797 %}
10798 
10799 // Immediate Subtraction
10800 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10801   match(Set dst (SubI src1 src2));
10802 
10803   ins_cost(INSN_COST);
10804   format %{ "subw $dst, $src1, $src2" %}
10805 
10806   // use opcode to indicate that this is a sub not an add
10807   opcode(0x1);
10808 
10809   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10810 
10811   ins_pipe(ialu_reg_imm);
10812 %}
10813 
10814 // Long Subtraction
10815 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10816 
10817   match(Set dst (SubL src1 src2));
10818 
10819   ins_cost(INSN_COST);
10820   format %{ "sub  $dst, $src1, $src2" %}
10821 
10822   ins_encode %{
10823     __ sub(as_Register($dst$$reg),
10824            as_Register($src1$$reg),
10825            as_Register($src2$$reg));
10826   %}
10827 
10828   ins_pipe(ialu_reg_reg);
10829 %}
10830 
10831 // No constant pool entries requiredLong Immediate Subtraction.
10832 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10833   match(Set dst (SubL src1 src2));
10834 
10835   ins_cost(INSN_COST);
10836   format %{ "sub$dst, $src1, $src2" %}
10837 
10838   // use opcode to indicate that this is a sub not an add
10839   opcode(0x1);
10840 
10841   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10842 
10843   ins_pipe(ialu_reg_imm);
10844 %}
10845 
10846 // Integer Negation (special case for sub)
10847 
10848 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10849   match(Set dst (SubI zero src));
10850 
10851   ins_cost(INSN_COST);
10852   format %{ "negw $dst, $src\t# int" %}
10853 
10854   ins_encode %{
10855     __ negw(as_Register($dst$$reg),
10856             as_Register($src$$reg));
10857   %}
10858 
10859   ins_pipe(ialu_reg);
10860 %}
10861 
10862 // Long Negation
10863 
10864 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10865   match(Set dst (SubL zero src));
10866 
10867   ins_cost(INSN_COST);
10868   format %{ "neg $dst, $src\t# long" %}
10869 
10870   ins_encode %{
10871     __ neg(as_Register($dst$$reg),
10872            as_Register($src$$reg));
10873   %}
10874 
10875   ins_pipe(ialu_reg);
10876 %}
10877 
10878 // Integer Multiply
10879 
10880 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10881   match(Set dst (MulI src1 src2));
10882 
10883   ins_cost(INSN_COST * 3);
10884   format %{ "mulw  $dst, $src1, $src2" %}
10885 
10886   ins_encode %{
10887     __ mulw(as_Register($dst$$reg),
10888             as_Register($src1$$reg),
10889             as_Register($src2$$reg));
10890   %}
10891 
10892   ins_pipe(imul_reg_reg);
10893 %}
10894 
10895 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10896   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10897 
10898   ins_cost(INSN_COST * 3);
10899   format %{ "smull  $dst, $src1, $src2" %}
10900 
10901   ins_encode %{
10902     __ smull(as_Register($dst$$reg),
10903              as_Register($src1$$reg),
10904              as_Register($src2$$reg));
10905   %}
10906 
10907   ins_pipe(imul_reg_reg);
10908 %}
10909 
10910 // Long Multiply
10911 
10912 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10913   match(Set dst (MulL src1 src2));
10914 
10915   ins_cost(INSN_COST * 5);
10916   format %{ "mul  $dst, $src1, $src2" %}
10917 
10918   ins_encode %{
10919     __ mul(as_Register($dst$$reg),
10920            as_Register($src1$$reg),
10921            as_Register($src2$$reg));
10922   %}
10923 
10924   ins_pipe(lmul_reg_reg);
10925 %}
10926 
10927 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10928 %{
10929   match(Set dst (MulHiL src1 src2));
10930 
10931   ins_cost(INSN_COST * 7);
10932   format %{ "smulh   $dst, $src1, $src2, \t# mulhi" %}
10933 
10934   ins_encode %{
10935     __ smulh(as_Register($dst$$reg),
10936              as_Register($src1$$reg),
10937              as_Register($src2$$reg));
10938   %}
10939 
10940   ins_pipe(lmul_reg_reg);
10941 %}
10942 
10943 // Combined Integer Multiply & Add/Sub
10944 
10945 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10946   match(Set dst (AddI src3 (MulI src1 src2)));
10947 
10948   ins_cost(INSN_COST * 3);
10949   format %{ "madd  $dst, $src1, $src2, $src3" %}
10950 
10951   ins_encode %{
10952     __ maddw(as_Register($dst$$reg),
10953              as_Register($src1$$reg),
10954              as_Register($src2$$reg),
10955              as_Register($src3$$reg));
10956   %}
10957 
10958   ins_pipe(imac_reg_reg);
10959 %}
10960 
10961 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10962   match(Set dst (SubI src3 (MulI src1 src2)));
10963 
10964   ins_cost(INSN_COST * 3);
10965   format %{ "msub  $dst, $src1, $src2, $src3" %}
10966 
10967   ins_encode %{
10968     __ msubw(as_Register($dst$$reg),
10969              as_Register($src1$$reg),
10970              as_Register($src2$$reg),
10971              as_Register($src3$$reg));
10972   %}
10973 
10974   ins_pipe(imac_reg_reg);
10975 %}
10976 
10977 // Combined Integer Multiply & Neg
10978 
10979 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10980   match(Set dst (MulI (SubI zero src1) src2));
10981   match(Set dst (MulI src1 (SubI zero src2)));
10982 
10983   ins_cost(INSN_COST * 3);
10984   format %{ "mneg  $dst, $src1, $src2" %}
10985 
10986   ins_encode %{
10987     __ mnegw(as_Register($dst$$reg),
10988              as_Register($src1$$reg),
10989              as_Register($src2$$reg));
10990   %}
10991 
10992   ins_pipe(imac_reg_reg);
10993 %}
10994 
10995 // Combined Long Multiply & Add/Sub
10996 
10997 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10998   match(Set dst (AddL src3 (MulL src1 src2)));
10999 
11000   ins_cost(INSN_COST * 5);
11001   format %{ "madd  $dst, $src1, $src2, $src3" %}
11002 
11003   ins_encode %{
11004     __ madd(as_Register($dst$$reg),
11005             as_Register($src1$$reg),
11006             as_Register($src2$$reg),
11007             as_Register($src3$$reg));
11008   %}
11009 
11010   ins_pipe(lmac_reg_reg);
11011 %}
11012 
11013 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11014   match(Set dst (SubL src3 (MulL src1 src2)));
11015 
11016   ins_cost(INSN_COST * 5);
11017   format %{ "msub  $dst, $src1, $src2, $src3" %}
11018 
11019   ins_encode %{
11020     __ msub(as_Register($dst$$reg),
11021             as_Register($src1$$reg),
11022             as_Register($src2$$reg),
11023             as_Register($src3$$reg));
11024   %}
11025 
11026   ins_pipe(lmac_reg_reg);
11027 %}
11028 
11029 // Combined Long Multiply & Neg
11030 
11031 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11032   match(Set dst (MulL (SubL zero src1) src2));
11033   match(Set dst (MulL src1 (SubL zero src2)));
11034 
11035   ins_cost(INSN_COST * 5);
11036   format %{ "mneg  $dst, $src1, $src2" %}
11037 
11038   ins_encode %{
11039     __ mneg(as_Register($dst$$reg),
11040             as_Register($src1$$reg),
11041             as_Register($src2$$reg));
11042   %}
11043 
11044   ins_pipe(lmac_reg_reg);
11045 %}
11046 
11047 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11048 
11049 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11050   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11051 
11052   ins_cost(INSN_COST * 3);
11053   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
11054 
11055   ins_encode %{
11056     __ smaddl(as_Register($dst$$reg),
11057               as_Register($src1$$reg),
11058               as_Register($src2$$reg),
11059               as_Register($src3$$reg));
11060   %}
11061 
11062   ins_pipe(imac_reg_reg);
11063 %}
11064 
11065 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11066   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11067 
11068   ins_cost(INSN_COST * 3);
11069   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
11070 
11071   ins_encode %{
11072     __ smsubl(as_Register($dst$$reg),
11073               as_Register($src1$$reg),
11074               as_Register($src2$$reg),
11075               as_Register($src3$$reg));
11076   %}
11077 
11078   ins_pipe(imac_reg_reg);
11079 %}
11080 
11081 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11082   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11083   match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11084 
11085   ins_cost(INSN_COST * 3);
11086   format %{ "smnegl  $dst, $src1, $src2" %}
11087 
11088   ins_encode %{
11089     __ smnegl(as_Register($dst$$reg),
11090               as_Register($src1$$reg),
11091               as_Register($src2$$reg));
11092   %}
11093 
11094   ins_pipe(imac_reg_reg);
11095 %}
11096 
11097 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11098 
11099 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11100   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11101 
11102   ins_cost(INSN_COST * 5);
11103   format %{ "mulw  rscratch1, $src1, $src2\n\t"
11104             "maddw $dst, $src3, $src4, rscratch1" %}
11105 
11106   ins_encode %{
11107     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11108     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11109 
11110   ins_pipe(imac_reg_reg);
11111 %}
11112 
11113 // Integer Divide
11114 
11115 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11116   match(Set dst (DivI src1 src2));
11117 
11118   ins_cost(INSN_COST * 19);
11119   format %{ "sdivw  $dst, $src1, $src2" %}
11120 
11121   ins_encode(aarch64_enc_divw(dst, src1, src2));
11122   ins_pipe(idiv_reg_reg);
11123 %}
11124 
11125 // Long Divide
11126 
11127 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11128   match(Set dst (DivL src1 src2));
11129 
11130   ins_cost(INSN_COST * 35);
11131   format %{ "sdiv   $dst, $src1, $src2" %}
11132 
11133   ins_encode(aarch64_enc_div(dst, src1, src2));
11134   ins_pipe(ldiv_reg_reg);
11135 %}
11136 
11137 // Integer Remainder
11138 
11139 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11140   match(Set dst (ModI src1 src2));
11141 
11142   ins_cost(INSN_COST * 22);
11143   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11144             "msubw($dst, rscratch1, $src2, $src1" %}
11145 
11146   ins_encode(aarch64_enc_modw(dst, src1, src2));
11147   ins_pipe(idiv_reg_reg);
11148 %}
11149 
11150 // Long Remainder
11151 
11152 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11153   match(Set dst (ModL src1 src2));
11154 
11155   ins_cost(INSN_COST * 38);
11156   format %{ "sdiv   rscratch1, $src1, $src2\n"
11157             "msub($dst, rscratch1, $src2, $src1" %}
11158 
11159   ins_encode(aarch64_enc_mod(dst, src1, src2));
11160   ins_pipe(ldiv_reg_reg);
11161 %}
11162 
11163 // Integer Shifts
11164 
11165 // Shift Left Register
11166 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11167   match(Set dst (LShiftI src1 src2));
11168 
11169   ins_cost(INSN_COST * 2);
11170   format %{ "lslvw  $dst, $src1, $src2" %}
11171 
11172   ins_encode %{
11173     __ lslvw(as_Register($dst$$reg),
11174              as_Register($src1$$reg),
11175              as_Register($src2$$reg));
11176   %}
11177 
11178   ins_pipe(ialu_reg_reg_vshift);
11179 %}
11180 
11181 // Shift Left Immediate
11182 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11183   match(Set dst (LShiftI src1 src2));
11184 
11185   ins_cost(INSN_COST);
11186   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11187 
11188   ins_encode %{
11189     __ lslw(as_Register($dst$$reg),
11190             as_Register($src1$$reg),
11191             $src2$$constant & 0x1f);
11192   %}
11193 
11194   ins_pipe(ialu_reg_shift);
11195 %}
11196 
11197 // Shift Right Logical Register
11198 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11199   match(Set dst (URShiftI src1 src2));
11200 
11201   ins_cost(INSN_COST * 2);
11202   format %{ "lsrvw  $dst, $src1, $src2" %}
11203 
11204   ins_encode %{
11205     __ lsrvw(as_Register($dst$$reg),
11206              as_Register($src1$$reg),
11207              as_Register($src2$$reg));
11208   %}
11209 
11210   ins_pipe(ialu_reg_reg_vshift);
11211 %}
11212 
11213 // Shift Right Logical Immediate
11214 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11215   match(Set dst (URShiftI src1 src2));
11216 
11217   ins_cost(INSN_COST);
11218   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11219 
11220   ins_encode %{
11221     __ lsrw(as_Register($dst$$reg),
11222             as_Register($src1$$reg),
11223             $src2$$constant & 0x1f);
11224   %}
11225 
11226   ins_pipe(ialu_reg_shift);
11227 %}
11228 
11229 // Shift Right Arithmetic Register
11230 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11231   match(Set dst (RShiftI src1 src2));
11232 
11233   ins_cost(INSN_COST * 2);
11234   format %{ "asrvw  $dst, $src1, $src2" %}
11235 
11236   ins_encode %{
11237     __ asrvw(as_Register($dst$$reg),
11238              as_Register($src1$$reg),
11239              as_Register($src2$$reg));
11240   %}
11241 
11242   ins_pipe(ialu_reg_reg_vshift);
11243 %}
11244 
11245 // Shift Right Arithmetic Immediate
11246 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11247   match(Set dst (RShiftI src1 src2));
11248 
11249   ins_cost(INSN_COST);
11250   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11251 
11252   ins_encode %{
11253     __ asrw(as_Register($dst$$reg),
11254             as_Register($src1$$reg),
11255             $src2$$constant & 0x1f);
11256   %}
11257 
11258   ins_pipe(ialu_reg_shift);
11259 %}
11260 
11261 // Combined Int Mask and Right Shift (using UBFM)
11262 // TODO
11263 
11264 // Long Shifts
11265 
11266 // Shift Left Register
11267 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11268   match(Set dst (LShiftL src1 src2));
11269 
11270   ins_cost(INSN_COST * 2);
11271   format %{ "lslv  $dst, $src1, $src2" %}
11272 
11273   ins_encode %{
11274     __ lslv(as_Register($dst$$reg),
11275             as_Register($src1$$reg),
11276             as_Register($src2$$reg));
11277   %}
11278 
11279   ins_pipe(ialu_reg_reg_vshift);
11280 %}
11281 
11282 // Shift Left Immediate
11283 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11284   match(Set dst (LShiftL src1 src2));
11285 
11286   ins_cost(INSN_COST);
11287   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11288 
11289   ins_encode %{
11290     __ lsl(as_Register($dst$$reg),
11291             as_Register($src1$$reg),
11292             $src2$$constant & 0x3f);
11293   %}
11294 
11295   ins_pipe(ialu_reg_shift);
11296 %}
11297 
11298 // Shift Right Logical Register
11299 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11300   match(Set dst (URShiftL src1 src2));
11301 
11302   ins_cost(INSN_COST * 2);
11303   format %{ "lsrv  $dst, $src1, $src2" %}
11304 
11305   ins_encode %{
11306     __ lsrv(as_Register($dst$$reg),
11307             as_Register($src1$$reg),
11308             as_Register($src2$$reg));
11309   %}
11310 
11311   ins_pipe(ialu_reg_reg_vshift);
11312 %}
11313 
11314 // Shift Right Logical Immediate
11315 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11316   match(Set dst (URShiftL src1 src2));
11317 
11318   ins_cost(INSN_COST);
11319   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11320 
11321   ins_encode %{
11322     __ lsr(as_Register($dst$$reg),
11323            as_Register($src1$$reg),
11324            $src2$$constant & 0x3f);
11325   %}
11326 
11327   ins_pipe(ialu_reg_shift);
11328 %}
11329 
11330 // A special-case pattern for card table stores.
11331 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11332   match(Set dst (URShiftL (CastP2X src1) src2));
11333 
11334   ins_cost(INSN_COST);
11335   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11336 
11337   ins_encode %{
11338     __ lsr(as_Register($dst$$reg),
11339            as_Register($src1$$reg),
11340            $src2$$constant & 0x3f);
11341   %}
11342 
11343   ins_pipe(ialu_reg_shift);
11344 %}
11345 
11346 // Shift Right Arithmetic Register
11347 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11348   match(Set dst (RShiftL src1 src2));
11349 
11350   ins_cost(INSN_COST * 2);
11351   format %{ "asrv  $dst, $src1, $src2" %}
11352 
11353   ins_encode %{
11354     __ asrv(as_Register($dst$$reg),
11355             as_Register($src1$$reg),
11356             as_Register($src2$$reg));
11357   %}
11358 
11359   ins_pipe(ialu_reg_reg_vshift);
11360 %}
11361 
11362 // Shift Right Arithmetic Immediate
11363 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11364   match(Set dst (RShiftL src1 src2));
11365 
11366   ins_cost(INSN_COST);
11367   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11368 
11369   ins_encode %{
11370     __ asr(as_Register($dst$$reg),
11371            as_Register($src1$$reg),
11372            $src2$$constant & 0x3f);
11373   %}
11374 
11375   ins_pipe(ialu_reg_shift);
11376 %}
11377 
11378 // BEGIN This section of the file is automatically generated. Do not edit --------------
11379 // This section is generated from aarch64_ad.m4
11380 
11381 
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 instruct regL_not_reg(iRegLNoSp dst,
11385                          iRegL src1, immL_M1 m1,
11386                          rFlagsReg cr) %{
11387   match(Set dst (XorL src1 m1));
11388   ins_cost(INSN_COST);
11389   format %{ "eon  $dst, $src1, zr" %}
11390 
11391   ins_encode %{
11392     __ eon(as_Register($dst$$reg),
11393               as_Register($src1$$reg),
11394               zr,
11395               Assembler::LSL, 0);
11396   %}
11397 
11398   ins_pipe(ialu_reg);
11399 %}
11400 
11401 // This pattern is automatically generated from aarch64_ad.m4
11402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11403 instruct regI_not_reg(iRegINoSp dst,
11404                          iRegIorL2I src1, immI_M1 m1,
11405                          rFlagsReg cr) %{
11406   match(Set dst (XorI src1 m1));
11407   ins_cost(INSN_COST);
11408   format %{ "eonw  $dst, $src1, zr" %}
11409 
11410   ins_encode %{
11411     __ eonw(as_Register($dst$$reg),
11412               as_Register($src1$$reg),
11413               zr,
11414               Assembler::LSL, 0);
11415   %}
11416 
11417   ins_pipe(ialu_reg);
11418 %}
11419 
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11423                               immI0 zero, iRegIorL2I src1, immI src2) %{
11424   match(Set dst (SubI zero (URShiftI src1 src2)));
11425 
11426   ins_cost(1.9 * INSN_COST);
11427   format %{ "negw  $dst, $src1, LSR $src2" %}
11428 
11429   ins_encode %{
11430     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11431             Assembler::LSR, $src2$$constant & 0x1f);
11432   %}
11433 
11434   ins_pipe(ialu_reg_shift);
11435 %}
11436 
11437 // This pattern is automatically generated from aarch64_ad.m4
11438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11439 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11440                               immI0 zero, iRegIorL2I src1, immI src2) %{
11441   match(Set dst (SubI zero (RShiftI src1 src2)));
11442 
11443   ins_cost(1.9 * INSN_COST);
11444   format %{ "negw  $dst, $src1, ASR $src2" %}
11445 
11446   ins_encode %{
11447     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11448             Assembler::ASR, $src2$$constant & 0x1f);
11449   %}
11450 
11451   ins_pipe(ialu_reg_shift);
11452 %}
11453 
11454 // This pattern is automatically generated from aarch64_ad.m4
11455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11456 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11457                               immI0 zero, iRegIorL2I src1, immI src2) %{
11458   match(Set dst (SubI zero (LShiftI src1 src2)));
11459 
11460   ins_cost(1.9 * INSN_COST);
11461   format %{ "negw  $dst, $src1, LSL $src2" %}
11462 
11463   ins_encode %{
11464     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11465             Assembler::LSL, $src2$$constant & 0x1f);
11466   %}
11467 
11468   ins_pipe(ialu_reg_shift);
11469 %}
11470 
11471 // This pattern is automatically generated from aarch64_ad.m4
11472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11473 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11474                               immL0 zero, iRegL src1, immI src2) %{
11475   match(Set dst (SubL zero (URShiftL src1 src2)));
11476 
11477   ins_cost(1.9 * INSN_COST);
11478   format %{ "neg  $dst, $src1, LSR $src2" %}
11479 
11480   ins_encode %{
11481     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11482             Assembler::LSR, $src2$$constant & 0x3f);
11483   %}
11484 
11485   ins_pipe(ialu_reg_shift);
11486 %}
11487 
11488 // This pattern is automatically generated from aarch64_ad.m4
11489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11490 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11491                               immL0 zero, iRegL src1, immI src2) %{
11492   match(Set dst (SubL zero (RShiftL src1 src2)));
11493 
11494   ins_cost(1.9 * INSN_COST);
11495   format %{ "neg  $dst, $src1, ASR $src2" %}
11496 
11497   ins_encode %{
11498     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11499             Assembler::ASR, $src2$$constant & 0x3f);
11500   %}
11501 
11502   ins_pipe(ialu_reg_shift);
11503 %}
11504 
11505 // This pattern is automatically generated from aarch64_ad.m4
11506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11507 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11508                               immL0 zero, iRegL src1, immI src2) %{
11509   match(Set dst (SubL zero (LShiftL src1 src2)));
11510 
11511   ins_cost(1.9 * INSN_COST);
11512   format %{ "neg  $dst, $src1, LSL $src2" %}
11513 
11514   ins_encode %{
11515     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11516             Assembler::LSL, $src2$$constant & 0x3f);
11517   %}
11518 
11519   ins_pipe(ialu_reg_shift);
11520 %}
11521 
11522 // This pattern is automatically generated from aarch64_ad.m4
11523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11524 instruct AndI_reg_not_reg(iRegINoSp dst,
11525                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11526   match(Set dst (AndI src1 (XorI src2 m1)));
11527   ins_cost(INSN_COST);
11528   format %{ "bicw  $dst, $src1, $src2" %}
11529 
11530   ins_encode %{
11531     __ bicw(as_Register($dst$$reg),
11532               as_Register($src1$$reg),
11533               as_Register($src2$$reg),
11534               Assembler::LSL, 0);
11535   %}
11536 
11537   ins_pipe(ialu_reg_reg);
11538 %}
11539 
11540 // This pattern is automatically generated from aarch64_ad.m4
11541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11542 instruct AndL_reg_not_reg(iRegLNoSp dst,
11543                          iRegL src1, iRegL src2, immL_M1 m1) %{
11544   match(Set dst (AndL src1 (XorL src2 m1)));
11545   ins_cost(INSN_COST);
11546   format %{ "bic  $dst, $src1, $src2" %}
11547 
11548   ins_encode %{
11549     __ bic(as_Register($dst$$reg),
11550               as_Register($src1$$reg),
11551               as_Register($src2$$reg),
11552               Assembler::LSL, 0);
11553   %}
11554 
11555   ins_pipe(ialu_reg_reg);
11556 %}
11557 
11558 // This pattern is automatically generated from aarch64_ad.m4
11559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11560 instruct OrI_reg_not_reg(iRegINoSp dst,
11561                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11562   match(Set dst (OrI src1 (XorI src2 m1)));
11563   ins_cost(INSN_COST);
11564   format %{ "ornw  $dst, $src1, $src2" %}
11565 
11566   ins_encode %{
11567     __ ornw(as_Register($dst$$reg),
11568               as_Register($src1$$reg),
11569               as_Register($src2$$reg),
11570               Assembler::LSL, 0);
11571   %}
11572 
11573   ins_pipe(ialu_reg_reg);
11574 %}
11575 
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 instruct OrL_reg_not_reg(iRegLNoSp dst,
11579                          iRegL src1, iRegL src2, immL_M1 m1) %{
11580   match(Set dst (OrL src1 (XorL src2 m1)));
11581   ins_cost(INSN_COST);
11582   format %{ "orn  $dst, $src1, $src2" %}
11583 
11584   ins_encode %{
11585     __ orn(as_Register($dst$$reg),
11586               as_Register($src1$$reg),
11587               as_Register($src2$$reg),
11588               Assembler::LSL, 0);
11589   %}
11590 
11591   ins_pipe(ialu_reg_reg);
11592 %}
11593 
11594 // This pattern is automatically generated from aarch64_ad.m4
11595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11596 instruct XorI_reg_not_reg(iRegINoSp dst,
11597                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11598   match(Set dst (XorI m1 (XorI src2 src1)));
11599   ins_cost(INSN_COST);
11600   format %{ "eonw  $dst, $src1, $src2" %}
11601 
11602   ins_encode %{
11603     __ eonw(as_Register($dst$$reg),
11604               as_Register($src1$$reg),
11605               as_Register($src2$$reg),
11606               Assembler::LSL, 0);
11607   %}
11608 
11609   ins_pipe(ialu_reg_reg);
11610 %}
11611 
11612 // This pattern is automatically generated from aarch64_ad.m4
11613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11614 instruct XorL_reg_not_reg(iRegLNoSp dst,
11615                          iRegL src1, iRegL src2, immL_M1 m1) %{
11616   match(Set dst (XorL m1 (XorL src2 src1)));
11617   ins_cost(INSN_COST);
11618   format %{ "eon  $dst, $src1, $src2" %}
11619 
11620   ins_encode %{
11621     __ eon(as_Register($dst$$reg),
11622               as_Register($src1$$reg),
11623               as_Register($src2$$reg),
11624               Assembler::LSL, 0);
11625   %}
11626 
11627   ins_pipe(ialu_reg_reg);
11628 %}
11629 
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 // val & (-1 ^ (val >>> shift)) ==> bicw
11633 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11634                          iRegIorL2I src1, iRegIorL2I src2,
11635                          immI src3, immI_M1 src4) %{
11636   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11637   ins_cost(1.9 * INSN_COST);
11638   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11639 
11640   ins_encode %{
11641     __ bicw(as_Register($dst$$reg),
11642               as_Register($src1$$reg),
11643               as_Register($src2$$reg),
11644               Assembler::LSR,
11645               $src3$$constant & 0x1f);
11646   %}
11647 
11648   ins_pipe(ialu_reg_reg_shift);
11649 %}
11650 
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653 // val & (-1 ^ (val >>> shift)) ==> bic
11654 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11655                          iRegL src1, iRegL src2,
11656                          immI src3, immL_M1 src4) %{
11657   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11658   ins_cost(1.9 * INSN_COST);
11659   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11660 
11661   ins_encode %{
11662     __ bic(as_Register($dst$$reg),
11663               as_Register($src1$$reg),
11664               as_Register($src2$$reg),
11665               Assembler::LSR,
11666               $src3$$constant & 0x3f);
11667   %}
11668 
11669   ins_pipe(ialu_reg_reg_shift);
11670 %}
11671 
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 // val & (-1 ^ (val >> shift)) ==> bicw
11675 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11676                          iRegIorL2I src1, iRegIorL2I src2,
11677                          immI src3, immI_M1 src4) %{
11678   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11679   ins_cost(1.9 * INSN_COST);
11680   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11681 
11682   ins_encode %{
11683     __ bicw(as_Register($dst$$reg),
11684               as_Register($src1$$reg),
11685               as_Register($src2$$reg),
11686               Assembler::ASR,
11687               $src3$$constant & 0x1f);
11688   %}
11689 
11690   ins_pipe(ialu_reg_reg_shift);
11691 %}
11692 
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695 // val & (-1 ^ (val >> shift)) ==> bic
11696 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11697                          iRegL src1, iRegL src2,
11698                          immI src3, immL_M1 src4) %{
11699   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11700   ins_cost(1.9 * INSN_COST);
11701   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11702 
11703   ins_encode %{
11704     __ bic(as_Register($dst$$reg),
11705               as_Register($src1$$reg),
11706               as_Register($src2$$reg),
11707               Assembler::ASR,
11708               $src3$$constant & 0x3f);
11709   %}
11710 
11711   ins_pipe(ialu_reg_reg_shift);
11712 %}
11713 
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 // val & (-1 ^ (val ror shift)) ==> bicw
11717 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11718                          iRegIorL2I src1, iRegIorL2I src2,
11719                          immI src3, immI_M1 src4) %{
11720   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11721   ins_cost(1.9 * INSN_COST);
11722   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11723 
11724   ins_encode %{
11725     __ bicw(as_Register($dst$$reg),
11726               as_Register($src1$$reg),
11727               as_Register($src2$$reg),
11728               Assembler::ROR,
11729               $src3$$constant & 0x1f);
11730   %}
11731 
11732   ins_pipe(ialu_reg_reg_shift);
11733 %}
11734 
11735 // This pattern is automatically generated from aarch64_ad.m4
11736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11737 // val & (-1 ^ (val ror shift)) ==> bic
11738 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11739                          iRegL src1, iRegL src2,
11740                          immI src3, immL_M1 src4) %{
11741   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11742   ins_cost(1.9 * INSN_COST);
11743   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11744 
11745   ins_encode %{
11746     __ bic(as_Register($dst$$reg),
11747               as_Register($src1$$reg),
11748               as_Register($src2$$reg),
11749               Assembler::ROR,
11750               $src3$$constant & 0x3f);
11751   %}
11752 
11753   ins_pipe(ialu_reg_reg_shift);
11754 %}
11755 
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 // val & (-1 ^ (val << shift)) ==> bicw
11759 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11760                          iRegIorL2I src1, iRegIorL2I src2,
11761                          immI src3, immI_M1 src4) %{
11762   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11763   ins_cost(1.9 * INSN_COST);
11764   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11765 
11766   ins_encode %{
11767     __ bicw(as_Register($dst$$reg),
11768               as_Register($src1$$reg),
11769               as_Register($src2$$reg),
11770               Assembler::LSL,
11771               $src3$$constant & 0x1f);
11772   %}
11773 
11774   ins_pipe(ialu_reg_reg_shift);
11775 %}
11776 
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 // val & (-1 ^ (val << shift)) ==> bic
11780 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11781                          iRegL src1, iRegL src2,
11782                          immI src3, immL_M1 src4) %{
11783   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11784   ins_cost(1.9 * INSN_COST);
11785   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11786 
11787   ins_encode %{
11788     __ bic(as_Register($dst$$reg),
11789               as_Register($src1$$reg),
11790               as_Register($src2$$reg),
11791               Assembler::LSL,
11792               $src3$$constant & 0x3f);
11793   %}
11794 
11795   ins_pipe(ialu_reg_reg_shift);
11796 %}
11797 
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11801 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11802                          iRegIorL2I src1, iRegIorL2I src2,
11803                          immI src3, immI_M1 src4) %{
11804   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11805   ins_cost(1.9 * INSN_COST);
11806   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11807 
11808   ins_encode %{
11809     __ eonw(as_Register($dst$$reg),
11810               as_Register($src1$$reg),
11811               as_Register($src2$$reg),
11812               Assembler::LSR,
11813               $src3$$constant & 0x1f);
11814   %}
11815 
11816   ins_pipe(ialu_reg_reg_shift);
11817 %}
11818 
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 // val ^ (-1 ^ (val >>> shift)) ==> eon
11822 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11823                          iRegL src1, iRegL src2,
11824                          immI src3, immL_M1 src4) %{
11825   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11826   ins_cost(1.9 * INSN_COST);
11827   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11828 
11829   ins_encode %{
11830     __ eon(as_Register($dst$$reg),
11831               as_Register($src1$$reg),
11832               as_Register($src2$$reg),
11833               Assembler::LSR,
11834               $src3$$constant & 0x3f);
11835   %}
11836 
11837   ins_pipe(ialu_reg_reg_shift);
11838 %}
11839 
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 // val ^ (-1 ^ (val >> shift)) ==> eonw
11843 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11844                          iRegIorL2I src1, iRegIorL2I src2,
11845                          immI src3, immI_M1 src4) %{
11846   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11847   ins_cost(1.9 * INSN_COST);
11848   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11849 
11850   ins_encode %{
11851     __ eonw(as_Register($dst$$reg),
11852               as_Register($src1$$reg),
11853               as_Register($src2$$reg),
11854               Assembler::ASR,
11855               $src3$$constant & 0x1f);
11856   %}
11857 
11858   ins_pipe(ialu_reg_reg_shift);
11859 %}
11860 
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 // val ^ (-1 ^ (val >> shift)) ==> eon
11864 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11865                          iRegL src1, iRegL src2,
11866                          immI src3, immL_M1 src4) %{
11867   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11868   ins_cost(1.9 * INSN_COST);
11869   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11870 
11871   ins_encode %{
11872     __ eon(as_Register($dst$$reg),
11873               as_Register($src1$$reg),
11874               as_Register($src2$$reg),
11875               Assembler::ASR,
11876               $src3$$constant & 0x3f);
11877   %}
11878 
11879   ins_pipe(ialu_reg_reg_shift);
11880 %}
11881 
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 // val ^ (-1 ^ (val ror shift)) ==> eonw
11885 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11886                          iRegIorL2I src1, iRegIorL2I src2,
11887                          immI src3, immI_M1 src4) %{
11888   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11889   ins_cost(1.9 * INSN_COST);
11890   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11891 
11892   ins_encode %{
11893     __ eonw(as_Register($dst$$reg),
11894               as_Register($src1$$reg),
11895               as_Register($src2$$reg),
11896               Assembler::ROR,
11897               $src3$$constant & 0x1f);
11898   %}
11899 
11900   ins_pipe(ialu_reg_reg_shift);
11901 %}
11902 
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 // val ^ (-1 ^ (val ror shift)) ==> eon
11906 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11907                          iRegL src1, iRegL src2,
11908                          immI src3, immL_M1 src4) %{
11909   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11910   ins_cost(1.9 * INSN_COST);
11911   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11912 
11913   ins_encode %{
11914     __ eon(as_Register($dst$$reg),
11915               as_Register($src1$$reg),
11916               as_Register($src2$$reg),
11917               Assembler::ROR,
11918               $src3$$constant & 0x3f);
11919   %}
11920 
11921   ins_pipe(ialu_reg_reg_shift);
11922 %}
11923 
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 // val ^ (-1 ^ (val << shift)) ==> eonw
11927 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11928                          iRegIorL2I src1, iRegIorL2I src2,
11929                          immI src3, immI_M1 src4) %{
11930   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11931   ins_cost(1.9 * INSN_COST);
11932   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11933 
11934   ins_encode %{
11935     __ eonw(as_Register($dst$$reg),
11936               as_Register($src1$$reg),
11937               as_Register($src2$$reg),
11938               Assembler::LSL,
11939               $src3$$constant & 0x1f);
11940   %}
11941 
11942   ins_pipe(ialu_reg_reg_shift);
11943 %}
11944 
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 // val ^ (-1 ^ (val << shift)) ==> eon
11948 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11949                          iRegL src1, iRegL src2,
11950                          immI src3, immL_M1 src4) %{
11951   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11952   ins_cost(1.9 * INSN_COST);
11953   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11954 
11955   ins_encode %{
11956     __ eon(as_Register($dst$$reg),
11957               as_Register($src1$$reg),
11958               as_Register($src2$$reg),
11959               Assembler::LSL,
11960               $src3$$constant & 0x3f);
11961   %}
11962 
11963   ins_pipe(ialu_reg_reg_shift);
11964 %}
11965 
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 // val | (-1 ^ (val >>> shift)) ==> ornw
11969 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11970                          iRegIorL2I src1, iRegIorL2I src2,
11971                          immI src3, immI_M1 src4) %{
11972   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11973   ins_cost(1.9 * INSN_COST);
11974   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11975 
11976   ins_encode %{
11977     __ ornw(as_Register($dst$$reg),
11978               as_Register($src1$$reg),
11979               as_Register($src2$$reg),
11980               Assembler::LSR,
11981               $src3$$constant & 0x1f);
11982   %}
11983 
11984   ins_pipe(ialu_reg_reg_shift);
11985 %}
11986 
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 // val | (-1 ^ (val >>> shift)) ==> orn
11990 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11991                          iRegL src1, iRegL src2,
11992                          immI src3, immL_M1 src4) %{
11993   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11994   ins_cost(1.9 * INSN_COST);
11995   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11996 
11997   ins_encode %{
11998     __ orn(as_Register($dst$$reg),
11999               as_Register($src1$$reg),
12000               as_Register($src2$$reg),
12001               Assembler::LSR,
12002               $src3$$constant & 0x3f);
12003   %}
12004 
12005   ins_pipe(ialu_reg_reg_shift);
12006 %}
12007 
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 // val | (-1 ^ (val >> shift)) ==> ornw
12011 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12012                          iRegIorL2I src1, iRegIorL2I src2,
12013                          immI src3, immI_M1 src4) %{
12014   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12015   ins_cost(1.9 * INSN_COST);
12016   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
12017 
12018   ins_encode %{
12019     __ ornw(as_Register($dst$$reg),
12020               as_Register($src1$$reg),
12021               as_Register($src2$$reg),
12022               Assembler::ASR,
12023               $src3$$constant & 0x1f);
12024   %}
12025 
12026   ins_pipe(ialu_reg_reg_shift);
12027 %}
12028 
12029 // This pattern is automatically generated from aarch64_ad.m4
12030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12031 // val | (-1 ^ (val >> shift)) ==> orn
12032 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12033                          iRegL src1, iRegL src2,
12034                          immI src3, immL_M1 src4) %{
12035   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12036   ins_cost(1.9 * INSN_COST);
12037   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
12038 
12039   ins_encode %{
12040     __ orn(as_Register($dst$$reg),
12041               as_Register($src1$$reg),
12042               as_Register($src2$$reg),
12043               Assembler::ASR,
12044               $src3$$constant & 0x3f);
12045   %}
12046 
12047   ins_pipe(ialu_reg_reg_shift);
12048 %}
12049 
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 // val | (-1 ^ (val ror shift)) ==> ornw
12053 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12054                          iRegIorL2I src1, iRegIorL2I src2,
12055                          immI src3, immI_M1 src4) %{
12056   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12057   ins_cost(1.9 * INSN_COST);
12058   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
12059 
12060   ins_encode %{
12061     __ ornw(as_Register($dst$$reg),
12062               as_Register($src1$$reg),
12063               as_Register($src2$$reg),
12064               Assembler::ROR,
12065               $src3$$constant & 0x1f);
12066   %}
12067 
12068   ins_pipe(ialu_reg_reg_shift);
12069 %}
12070 
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 // val | (-1 ^ (val ror shift)) ==> orn
12074 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12075                          iRegL src1, iRegL src2,
12076                          immI src3, immL_M1 src4) %{
12077   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12078   ins_cost(1.9 * INSN_COST);
12079   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12080 
12081   ins_encode %{
12082     __ orn(as_Register($dst$$reg),
12083               as_Register($src1$$reg),
12084               as_Register($src2$$reg),
12085               Assembler::ROR,
12086               $src3$$constant & 0x3f);
12087   %}
12088 
12089   ins_pipe(ialu_reg_reg_shift);
12090 %}
12091 
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 // val | (-1 ^ (val << shift)) ==> ornw
12095 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12096                          iRegIorL2I src1, iRegIorL2I src2,
12097                          immI src3, immI_M1 src4) %{
12098   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12099   ins_cost(1.9 * INSN_COST);
12100   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12101 
12102   ins_encode %{
12103     __ ornw(as_Register($dst$$reg),
12104               as_Register($src1$$reg),
12105               as_Register($src2$$reg),
12106               Assembler::LSL,
12107               $src3$$constant & 0x1f);
12108   %}
12109 
12110   ins_pipe(ialu_reg_reg_shift);
12111 %}
12112 
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 // val | (-1 ^ (val << shift)) ==> orn
12116 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12117                          iRegL src1, iRegL src2,
12118                          immI src3, immL_M1 src4) %{
12119   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12120   ins_cost(1.9 * INSN_COST);
12121   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12122 
12123   ins_encode %{
12124     __ orn(as_Register($dst$$reg),
12125               as_Register($src1$$reg),
12126               as_Register($src2$$reg),
12127               Assembler::LSL,
12128               $src3$$constant & 0x3f);
12129   %}
12130 
12131   ins_pipe(ialu_reg_reg_shift);
12132 %}
12133 
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12137                          iRegIorL2I src1, iRegIorL2I src2,
12138                          immI src3) %{
12139   match(Set dst (AndI src1 (URShiftI src2 src3)));
12140 
12141   ins_cost(1.9 * INSN_COST);
12142   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12143 
12144   ins_encode %{
12145     __ andw(as_Register($dst$$reg),
12146               as_Register($src1$$reg),
12147               as_Register($src2$$reg),
12148               Assembler::LSR,
12149               $src3$$constant & 0x1f);
12150   %}
12151 
12152   ins_pipe(ialu_reg_reg_shift);
12153 %}
12154 
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12158                          iRegL src1, iRegL src2,
12159                          immI src3) %{
12160   match(Set dst (AndL src1 (URShiftL src2 src3)));
12161 
12162   ins_cost(1.9 * INSN_COST);
12163   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12164 
12165   ins_encode %{
12166     __ andr(as_Register($dst$$reg),
12167               as_Register($src1$$reg),
12168               as_Register($src2$$reg),
12169               Assembler::LSR,
12170               $src3$$constant & 0x3f);
12171   %}
12172 
12173   ins_pipe(ialu_reg_reg_shift);
12174 %}
12175 
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12179                          iRegIorL2I src1, iRegIorL2I src2,
12180                          immI src3) %{
12181   match(Set dst (AndI src1 (RShiftI src2 src3)));
12182 
12183   ins_cost(1.9 * INSN_COST);
12184   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12185 
12186   ins_encode %{
12187     __ andw(as_Register($dst$$reg),
12188               as_Register($src1$$reg),
12189               as_Register($src2$$reg),
12190               Assembler::ASR,
12191               $src3$$constant & 0x1f);
12192   %}
12193 
12194   ins_pipe(ialu_reg_reg_shift);
12195 %}
12196 
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12200                          iRegL src1, iRegL src2,
12201                          immI src3) %{
12202   match(Set dst (AndL src1 (RShiftL src2 src3)));
12203 
12204   ins_cost(1.9 * INSN_COST);
12205   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12206 
12207   ins_encode %{
12208     __ andr(as_Register($dst$$reg),
12209               as_Register($src1$$reg),
12210               as_Register($src2$$reg),
12211               Assembler::ASR,
12212               $src3$$constant & 0x3f);
12213   %}
12214 
12215   ins_pipe(ialu_reg_reg_shift);
12216 %}
12217 
12218 // This pattern is automatically generated from aarch64_ad.m4
12219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12220 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12221                          iRegIorL2I src1, iRegIorL2I src2,
12222                          immI src3) %{
12223   match(Set dst (AndI src1 (LShiftI src2 src3)));
12224 
12225   ins_cost(1.9 * INSN_COST);
12226   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12227 
12228   ins_encode %{
12229     __ andw(as_Register($dst$$reg),
12230               as_Register($src1$$reg),
12231               as_Register($src2$$reg),
12232               Assembler::LSL,
12233               $src3$$constant & 0x1f);
12234   %}
12235 
12236   ins_pipe(ialu_reg_reg_shift);
12237 %}
12238 
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12242                          iRegL src1, iRegL src2,
12243                          immI src3) %{
12244   match(Set dst (AndL src1 (LShiftL src2 src3)));
12245 
12246   ins_cost(1.9 * INSN_COST);
12247   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12248 
12249   ins_encode %{
12250     __ andr(as_Register($dst$$reg),
12251               as_Register($src1$$reg),
12252               as_Register($src2$$reg),
12253               Assembler::LSL,
12254               $src3$$constant & 0x3f);
12255   %}
12256 
12257   ins_pipe(ialu_reg_reg_shift);
12258 %}
12259 
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12263                          iRegIorL2I src1, iRegIorL2I src2,
12264                          immI src3) %{
12265   match(Set dst (AndI src1 (RotateRight src2 src3)));
12266 
12267   ins_cost(1.9 * INSN_COST);
12268   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12269 
12270   ins_encode %{
12271     __ andw(as_Register($dst$$reg),
12272               as_Register($src1$$reg),
12273               as_Register($src2$$reg),
12274               Assembler::ROR,
12275               $src3$$constant & 0x1f);
12276   %}
12277 
12278   ins_pipe(ialu_reg_reg_shift);
12279 %}
12280 
12281 // This pattern is automatically generated from aarch64_ad.m4
12282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12283 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12284                          iRegL src1, iRegL src2,
12285                          immI src3) %{
12286   match(Set dst (AndL src1 (RotateRight src2 src3)));
12287 
12288   ins_cost(1.9 * INSN_COST);
12289   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12290 
12291   ins_encode %{
12292     __ andr(as_Register($dst$$reg),
12293               as_Register($src1$$reg),
12294               as_Register($src2$$reg),
12295               Assembler::ROR,
12296               $src3$$constant & 0x3f);
12297   %}
12298 
12299   ins_pipe(ialu_reg_reg_shift);
12300 %}
12301 
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12305                          iRegIorL2I src1, iRegIorL2I src2,
12306                          immI src3) %{
12307   match(Set dst (XorI src1 (URShiftI src2 src3)));
12308 
12309   ins_cost(1.9 * INSN_COST);
12310   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12311 
12312   ins_encode %{
12313     __ eorw(as_Register($dst$$reg),
12314               as_Register($src1$$reg),
12315               as_Register($src2$$reg),
12316               Assembler::LSR,
12317               $src3$$constant & 0x1f);
12318   %}
12319 
12320   ins_pipe(ialu_reg_reg_shift);
12321 %}
12322 
12323 // This pattern is automatically generated from aarch64_ad.m4
12324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12325 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12326                          iRegL src1, iRegL src2,
12327                          immI src3) %{
12328   match(Set dst (XorL src1 (URShiftL src2 src3)));
12329 
12330   ins_cost(1.9 * INSN_COST);
12331   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12332 
12333   ins_encode %{
12334     __ eor(as_Register($dst$$reg),
12335               as_Register($src1$$reg),
12336               as_Register($src2$$reg),
12337               Assembler::LSR,
12338               $src3$$constant & 0x3f);
12339   %}
12340 
12341   ins_pipe(ialu_reg_reg_shift);
12342 %}
12343 
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12347                          iRegIorL2I src1, iRegIorL2I src2,
12348                          immI src3) %{
12349   match(Set dst (XorI src1 (RShiftI src2 src3)));
12350 
12351   ins_cost(1.9 * INSN_COST);
12352   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12353 
12354   ins_encode %{
12355     __ eorw(as_Register($dst$$reg),
12356               as_Register($src1$$reg),
12357               as_Register($src2$$reg),
12358               Assembler::ASR,
12359               $src3$$constant & 0x1f);
12360   %}
12361 
12362   ins_pipe(ialu_reg_reg_shift);
12363 %}
12364 
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12368                          iRegL src1, iRegL src2,
12369                          immI src3) %{
12370   match(Set dst (XorL src1 (RShiftL src2 src3)));
12371 
12372   ins_cost(1.9 * INSN_COST);
12373   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12374 
12375   ins_encode %{
12376     __ eor(as_Register($dst$$reg),
12377               as_Register($src1$$reg),
12378               as_Register($src2$$reg),
12379               Assembler::ASR,
12380               $src3$$constant & 0x3f);
12381   %}
12382 
12383   ins_pipe(ialu_reg_reg_shift);
12384 %}
12385 
12386 // This pattern is automatically generated from aarch64_ad.m4
12387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12388 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12389                          iRegIorL2I src1, iRegIorL2I src2,
12390                          immI src3) %{
12391   match(Set dst (XorI src1 (LShiftI src2 src3)));
12392 
12393   ins_cost(1.9 * INSN_COST);
12394   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12395 
12396   ins_encode %{
12397     __ eorw(as_Register($dst$$reg),
12398               as_Register($src1$$reg),
12399               as_Register($src2$$reg),
12400               Assembler::LSL,
12401               $src3$$constant & 0x1f);
12402   %}
12403 
12404   ins_pipe(ialu_reg_reg_shift);
12405 %}
12406 
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12410                          iRegL src1, iRegL src2,
12411                          immI src3) %{
12412   match(Set dst (XorL src1 (LShiftL src2 src3)));
12413 
12414   ins_cost(1.9 * INSN_COST);
12415   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12416 
12417   ins_encode %{
12418     __ eor(as_Register($dst$$reg),
12419               as_Register($src1$$reg),
12420               as_Register($src2$$reg),
12421               Assembler::LSL,
12422               $src3$$constant & 0x3f);
12423   %}
12424 
12425   ins_pipe(ialu_reg_reg_shift);
12426 %}
12427 
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12431                          iRegIorL2I src1, iRegIorL2I src2,
12432                          immI src3) %{
12433   match(Set dst (XorI src1 (RotateRight src2 src3)));
12434 
12435   ins_cost(1.9 * INSN_COST);
12436   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12437 
12438   ins_encode %{
12439     __ eorw(as_Register($dst$$reg),
12440               as_Register($src1$$reg),
12441               as_Register($src2$$reg),
12442               Assembler::ROR,
12443               $src3$$constant & 0x1f);
12444   %}
12445 
12446   ins_pipe(ialu_reg_reg_shift);
12447 %}
12448 
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12452                          iRegL src1, iRegL src2,
12453                          immI src3) %{
12454   match(Set dst (XorL src1 (RotateRight src2 src3)));
12455 
12456   ins_cost(1.9 * INSN_COST);
12457   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12458 
12459   ins_encode %{
12460     __ eor(as_Register($dst$$reg),
12461               as_Register($src1$$reg),
12462               as_Register($src2$$reg),
12463               Assembler::ROR,
12464               $src3$$constant & 0x3f);
12465   %}
12466 
12467   ins_pipe(ialu_reg_reg_shift);
12468 %}
12469 
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12473                          iRegIorL2I src1, iRegIorL2I src2,
12474                          immI src3) %{
12475   match(Set dst (OrI src1 (URShiftI src2 src3)));
12476 
12477   ins_cost(1.9 * INSN_COST);
12478   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12479 
12480   ins_encode %{
12481     __ orrw(as_Register($dst$$reg),
12482               as_Register($src1$$reg),
12483               as_Register($src2$$reg),
12484               Assembler::LSR,
12485               $src3$$constant & 0x1f);
12486   %}
12487 
12488   ins_pipe(ialu_reg_reg_shift);
12489 %}
12490 
12491 // This pattern is automatically generated from aarch64_ad.m4
12492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12493 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12494                          iRegL src1, iRegL src2,
12495                          immI src3) %{
12496   match(Set dst (OrL src1 (URShiftL src2 src3)));
12497 
12498   ins_cost(1.9 * INSN_COST);
12499   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12500 
12501   ins_encode %{
12502     __ orr(as_Register($dst$$reg),
12503               as_Register($src1$$reg),
12504               as_Register($src2$$reg),
12505               Assembler::LSR,
12506               $src3$$constant & 0x3f);
12507   %}
12508 
12509   ins_pipe(ialu_reg_reg_shift);
12510 %}
12511 
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12515                          iRegIorL2I src1, iRegIorL2I src2,
12516                          immI src3) %{
12517   match(Set dst (OrI src1 (RShiftI src2 src3)));
12518 
12519   ins_cost(1.9 * INSN_COST);
12520   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12521 
12522   ins_encode %{
12523     __ orrw(as_Register($dst$$reg),
12524               as_Register($src1$$reg),
12525               as_Register($src2$$reg),
12526               Assembler::ASR,
12527               $src3$$constant & 0x1f);
12528   %}
12529 
12530   ins_pipe(ialu_reg_reg_shift);
12531 %}
12532 
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12536                          iRegL src1, iRegL src2,
12537                          immI src3) %{
12538   match(Set dst (OrL src1 (RShiftL src2 src3)));
12539 
12540   ins_cost(1.9 * INSN_COST);
12541   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12542 
12543   ins_encode %{
12544     __ orr(as_Register($dst$$reg),
12545               as_Register($src1$$reg),
12546               as_Register($src2$$reg),
12547               Assembler::ASR,
12548               $src3$$constant & 0x3f);
12549   %}
12550 
12551   ins_pipe(ialu_reg_reg_shift);
12552 %}
12553 
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12557                          iRegIorL2I src1, iRegIorL2I src2,
12558                          immI src3) %{
12559   match(Set dst (OrI src1 (LShiftI src2 src3)));
12560 
12561   ins_cost(1.9 * INSN_COST);
12562   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12563 
12564   ins_encode %{
12565     __ orrw(as_Register($dst$$reg),
12566               as_Register($src1$$reg),
12567               as_Register($src2$$reg),
12568               Assembler::LSL,
12569               $src3$$constant & 0x1f);
12570   %}
12571 
12572   ins_pipe(ialu_reg_reg_shift);
12573 %}
12574 
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12578                          iRegL src1, iRegL src2,
12579                          immI src3) %{
12580   match(Set dst (OrL src1 (LShiftL src2 src3)));
12581 
12582   ins_cost(1.9 * INSN_COST);
12583   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12584 
12585   ins_encode %{
12586     __ orr(as_Register($dst$$reg),
12587               as_Register($src1$$reg),
12588               as_Register($src2$$reg),
12589               Assembler::LSL,
12590               $src3$$constant & 0x3f);
12591   %}
12592 
12593   ins_pipe(ialu_reg_reg_shift);
12594 %}
12595 
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12599                          iRegIorL2I src1, iRegIorL2I src2,
12600                          immI src3) %{
12601   match(Set dst (OrI src1 (RotateRight src2 src3)));
12602 
12603   ins_cost(1.9 * INSN_COST);
12604   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12605 
12606   ins_encode %{
12607     __ orrw(as_Register($dst$$reg),
12608               as_Register($src1$$reg),
12609               as_Register($src2$$reg),
12610               Assembler::ROR,
12611               $src3$$constant & 0x1f);
12612   %}
12613 
12614   ins_pipe(ialu_reg_reg_shift);
12615 %}
12616 
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12620                          iRegL src1, iRegL src2,
12621                          immI src3) %{
12622   match(Set dst (OrL src1 (RotateRight src2 src3)));
12623 
12624   ins_cost(1.9 * INSN_COST);
12625   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12626 
12627   ins_encode %{
12628     __ orr(as_Register($dst$$reg),
12629               as_Register($src1$$reg),
12630               as_Register($src2$$reg),
12631               Assembler::ROR,
12632               $src3$$constant & 0x3f);
12633   %}
12634 
12635   ins_pipe(ialu_reg_reg_shift);
12636 %}
12637 
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12641                          iRegIorL2I src1, iRegIorL2I src2,
12642                          immI src3) %{
12643   match(Set dst (AddI src1 (URShiftI src2 src3)));
12644 
12645   ins_cost(1.9 * INSN_COST);
12646   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12647 
12648   ins_encode %{
12649     __ addw(as_Register($dst$$reg),
12650               as_Register($src1$$reg),
12651               as_Register($src2$$reg),
12652               Assembler::LSR,
12653               $src3$$constant & 0x1f);
12654   %}
12655 
12656   ins_pipe(ialu_reg_reg_shift);
12657 %}
12658 
12659 // This pattern is automatically generated from aarch64_ad.m4
12660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12661 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12662                          iRegL src1, iRegL src2,
12663                          immI src3) %{
12664   match(Set dst (AddL src1 (URShiftL src2 src3)));
12665 
12666   ins_cost(1.9 * INSN_COST);
12667   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12668 
12669   ins_encode %{
12670     __ add(as_Register($dst$$reg),
12671               as_Register($src1$$reg),
12672               as_Register($src2$$reg),
12673               Assembler::LSR,
12674               $src3$$constant & 0x3f);
12675   %}
12676 
12677   ins_pipe(ialu_reg_reg_shift);
12678 %}
12679 
12680 // This pattern is automatically generated from aarch64_ad.m4
12681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12682 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12683                          iRegIorL2I src1, iRegIorL2I src2,
12684                          immI src3) %{
12685   match(Set dst (AddI src1 (RShiftI src2 src3)));
12686 
12687   ins_cost(1.9 * INSN_COST);
12688   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12689 
12690   ins_encode %{
12691     __ addw(as_Register($dst$$reg),
12692               as_Register($src1$$reg),
12693               as_Register($src2$$reg),
12694               Assembler::ASR,
12695               $src3$$constant & 0x1f);
12696   %}
12697 
12698   ins_pipe(ialu_reg_reg_shift);
12699 %}
12700 
12701 // This pattern is automatically generated from aarch64_ad.m4
12702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12703 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12704                          iRegL src1, iRegL src2,
12705                          immI src3) %{
12706   match(Set dst (AddL src1 (RShiftL src2 src3)));
12707 
12708   ins_cost(1.9 * INSN_COST);
12709   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12710 
12711   ins_encode %{
12712     __ add(as_Register($dst$$reg),
12713               as_Register($src1$$reg),
12714               as_Register($src2$$reg),
12715               Assembler::ASR,
12716               $src3$$constant & 0x3f);
12717   %}
12718 
12719   ins_pipe(ialu_reg_reg_shift);
12720 %}
12721 
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12725                          iRegIorL2I src1, iRegIorL2I src2,
12726                          immI src3) %{
12727   match(Set dst (AddI src1 (LShiftI src2 src3)));
12728 
12729   ins_cost(1.9 * INSN_COST);
12730   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12731 
12732   ins_encode %{
12733     __ addw(as_Register($dst$$reg),
12734               as_Register($src1$$reg),
12735               as_Register($src2$$reg),
12736               Assembler::LSL,
12737               $src3$$constant & 0x1f);
12738   %}
12739 
12740   ins_pipe(ialu_reg_reg_shift);
12741 %}
12742 
12743 // This pattern is automatically generated from aarch64_ad.m4
12744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12745 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12746                          iRegL src1, iRegL src2,
12747                          immI src3) %{
12748   match(Set dst (AddL src1 (LShiftL src2 src3)));
12749 
12750   ins_cost(1.9 * INSN_COST);
12751   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12752 
12753   ins_encode %{
12754     __ add(as_Register($dst$$reg),
12755               as_Register($src1$$reg),
12756               as_Register($src2$$reg),
12757               Assembler::LSL,
12758               $src3$$constant & 0x3f);
12759   %}
12760 
12761   ins_pipe(ialu_reg_reg_shift);
12762 %}
12763 
12764 // This pattern is automatically generated from aarch64_ad.m4
12765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12766 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12767                          iRegIorL2I src1, iRegIorL2I src2,
12768                          immI src3) %{
12769   match(Set dst (SubI src1 (URShiftI src2 src3)));
12770 
12771   ins_cost(1.9 * INSN_COST);
12772   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12773 
12774   ins_encode %{
12775     __ subw(as_Register($dst$$reg),
12776               as_Register($src1$$reg),
12777               as_Register($src2$$reg),
12778               Assembler::LSR,
12779               $src3$$constant & 0x1f);
12780   %}
12781 
12782   ins_pipe(ialu_reg_reg_shift);
12783 %}
12784 
12785 // This pattern is automatically generated from aarch64_ad.m4
12786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12787 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12788                          iRegL src1, iRegL src2,
12789                          immI src3) %{
12790   match(Set dst (SubL src1 (URShiftL src2 src3)));
12791 
12792   ins_cost(1.9 * INSN_COST);
12793   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12794 
12795   ins_encode %{
12796     __ sub(as_Register($dst$$reg),
12797               as_Register($src1$$reg),
12798               as_Register($src2$$reg),
12799               Assembler::LSR,
12800               $src3$$constant & 0x3f);
12801   %}
12802 
12803   ins_pipe(ialu_reg_reg_shift);
12804 %}
12805 
12806 // This pattern is automatically generated from aarch64_ad.m4
12807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12808 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12809                          iRegIorL2I src1, iRegIorL2I src2,
12810                          immI src3) %{
12811   match(Set dst (SubI src1 (RShiftI src2 src3)));
12812 
12813   ins_cost(1.9 * INSN_COST);
12814   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12815 
12816   ins_encode %{
12817     __ subw(as_Register($dst$$reg),
12818               as_Register($src1$$reg),
12819               as_Register($src2$$reg),
12820               Assembler::ASR,
12821               $src3$$constant & 0x1f);
12822   %}
12823 
12824   ins_pipe(ialu_reg_reg_shift);
12825 %}
12826 
12827 // This pattern is automatically generated from aarch64_ad.m4
12828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12829 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12830                          iRegL src1, iRegL src2,
12831                          immI src3) %{
12832   match(Set dst (SubL src1 (RShiftL src2 src3)));
12833 
12834   ins_cost(1.9 * INSN_COST);
12835   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12836 
12837   ins_encode %{
12838     __ sub(as_Register($dst$$reg),
12839               as_Register($src1$$reg),
12840               as_Register($src2$$reg),
12841               Assembler::ASR,
12842               $src3$$constant & 0x3f);
12843   %}
12844 
12845   ins_pipe(ialu_reg_reg_shift);
12846 %}
12847 
12848 // This pattern is automatically generated from aarch64_ad.m4
12849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12850 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12851                          iRegIorL2I src1, iRegIorL2I src2,
12852                          immI src3) %{
12853   match(Set dst (SubI src1 (LShiftI src2 src3)));
12854 
12855   ins_cost(1.9 * INSN_COST);
12856   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12857 
12858   ins_encode %{
12859     __ subw(as_Register($dst$$reg),
12860               as_Register($src1$$reg),
12861               as_Register($src2$$reg),
12862               Assembler::LSL,
12863               $src3$$constant & 0x1f);
12864   %}
12865 
12866   ins_pipe(ialu_reg_reg_shift);
12867 %}
12868 
12869 // This pattern is automatically generated from aarch64_ad.m4
12870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12871 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12872                          iRegL src1, iRegL src2,
12873                          immI src3) %{
12874   match(Set dst (SubL src1 (LShiftL src2 src3)));
12875 
12876   ins_cost(1.9 * INSN_COST);
12877   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12878 
12879   ins_encode %{
12880     __ sub(as_Register($dst$$reg),
12881               as_Register($src1$$reg),
12882               as_Register($src2$$reg),
12883               Assembler::LSL,
12884               $src3$$constant & 0x3f);
12885   %}
12886 
12887   ins_pipe(ialu_reg_reg_shift);
12888 %}
12889 
12890 // This pattern is automatically generated from aarch64_ad.m4
12891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12892 
12893 // Shift Left followed by Shift Right.
12894 // This idiom is used by the compiler for the i2b bytecode etc.
12895 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12896 %{
12897   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12898   ins_cost(INSN_COST * 2);
12899   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12900   ins_encode %{
12901     int lshift = $lshift_count$$constant & 63;
12902     int rshift = $rshift_count$$constant & 63;
12903     int s = 63 - lshift;
12904     int r = (rshift - lshift) & 63;
12905     __ sbfm(as_Register($dst$$reg),
12906             as_Register($src$$reg),
12907             r, s);
12908   %}
12909 
12910   ins_pipe(ialu_reg_shift);
12911 %}
12912 
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 
12916 // Shift Left followed by Shift Right.
12917 // This idiom is used by the compiler for the i2b bytecode etc.
12918 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12919 %{
12920   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12921   ins_cost(INSN_COST * 2);
12922   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12923   ins_encode %{
12924     int lshift = $lshift_count$$constant & 31;
12925     int rshift = $rshift_count$$constant & 31;
12926     int s = 31 - lshift;
12927     int r = (rshift - lshift) & 31;
12928     __ sbfmw(as_Register($dst$$reg),
12929             as_Register($src$$reg),
12930             r, s);
12931   %}
12932 
12933   ins_pipe(ialu_reg_shift);
12934 %}
12935 
12936 // This pattern is automatically generated from aarch64_ad.m4
12937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12938 
12939 // Shift Left followed by Shift Right.
12940 // This idiom is used by the compiler for the i2b bytecode etc.
12941 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12942 %{
12943   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12944   ins_cost(INSN_COST * 2);
12945   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12946   ins_encode %{
12947     int lshift = $lshift_count$$constant & 63;
12948     int rshift = $rshift_count$$constant & 63;
12949     int s = 63 - lshift;
12950     int r = (rshift - lshift) & 63;
12951     __ ubfm(as_Register($dst$$reg),
12952             as_Register($src$$reg),
12953             r, s);
12954   %}
12955 
12956   ins_pipe(ialu_reg_shift);
12957 %}
12958 
12959 // This pattern is automatically generated from aarch64_ad.m4
12960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12961 
12962 // Shift Left followed by Shift Right.
12963 // This idiom is used by the compiler for the i2b bytecode etc.
12964 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12965 %{
12966   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12967   ins_cost(INSN_COST * 2);
12968   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12969   ins_encode %{
12970     int lshift = $lshift_count$$constant & 31;
12971     int rshift = $rshift_count$$constant & 31;
12972     int s = 31 - lshift;
12973     int r = (rshift - lshift) & 31;
12974     __ ubfmw(as_Register($dst$$reg),
12975             as_Register($src$$reg),
12976             r, s);
12977   %}
12978 
12979   ins_pipe(ialu_reg_shift);
12980 %}
12981 
12982 // Bitfield extract with shift & mask
12983 
12984 // This pattern is automatically generated from aarch64_ad.m4
12985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12986 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12987 %{
12988   match(Set dst (AndI (URShiftI src rshift) mask));
12989   // Make sure we are not going to exceed what ubfxw can do.
12990   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12991 
12992   ins_cost(INSN_COST);
12993   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12994   ins_encode %{
12995     int rshift = $rshift$$constant & 31;
12996     intptr_t mask = $mask$$constant;
12997     int width = exact_log2(mask+1);
12998     __ ubfxw(as_Register($dst$$reg),
12999             as_Register($src$$reg), rshift, width);
13000   %}
13001   ins_pipe(ialu_reg_shift);
13002 %}
13003 
13004 // This pattern is automatically generated from aarch64_ad.m4
13005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13006 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13007 %{
13008   match(Set dst (AndL (URShiftL src rshift) mask));
13009   // Make sure we are not going to exceed what ubfx can do.
13010   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13011 
13012   ins_cost(INSN_COST);
13013   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13014   ins_encode %{
13015     int rshift = $rshift$$constant & 63;
13016     intptr_t mask = $mask$$constant;
13017     int width = exact_log2_long(mask+1);
13018     __ ubfx(as_Register($dst$$reg),
13019             as_Register($src$$reg), rshift, width);
13020   %}
13021   ins_pipe(ialu_reg_shift);
13022 %}
13023 
13024 
13025 // This pattern is automatically generated from aarch64_ad.m4
13026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13027 
13028 // We can use ubfx when extending an And with a mask when we know mask
13029 // is positive.  We know that because immI_bitmask guarantees it.
13030 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13031 %{
13032   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13033   // Make sure we are not going to exceed what ubfxw can do.
13034   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13035 
13036   ins_cost(INSN_COST * 2);
13037   format %{ "ubfx $dst, $src, $rshift, $mask" %}
13038   ins_encode %{
13039     int rshift = $rshift$$constant & 31;
13040     intptr_t mask = $mask$$constant;
13041     int width = exact_log2(mask+1);
13042     __ ubfx(as_Register($dst$$reg),
13043             as_Register($src$$reg), rshift, width);
13044   %}
13045   ins_pipe(ialu_reg_shift);
13046 %}
13047 
13048 
13049 // This pattern is automatically generated from aarch64_ad.m4
13050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13051 
13052 // We can use ubfiz when masking by a positive number and then left shifting the result.
13053 // We know that the mask is positive because immI_bitmask guarantees it.
13054 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13055 %{
13056   match(Set dst (LShiftI (AndI src mask) lshift));
13057   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13058 
13059   ins_cost(INSN_COST);
13060   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13061   ins_encode %{
13062     int lshift = $lshift$$constant & 31;
13063     intptr_t mask = $mask$$constant;
13064     int width = exact_log2(mask+1);
13065     __ ubfizw(as_Register($dst$$reg),
13066           as_Register($src$$reg), lshift, width);
13067   %}
13068   ins_pipe(ialu_reg_shift);
13069 %}
13070 
13071 // This pattern is automatically generated from aarch64_ad.m4
13072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13073 
13074 // We can use ubfiz when masking by a positive number and then left shifting the result.
13075 // We know that the mask is positive because immL_bitmask guarantees it.
13076 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13077 %{
13078   match(Set dst (LShiftL (AndL src mask) lshift));
13079   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13080 
13081   ins_cost(INSN_COST);
13082   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13083   ins_encode %{
13084     int lshift = $lshift$$constant & 63;
13085     intptr_t mask = $mask$$constant;
13086     int width = exact_log2_long(mask+1);
13087     __ ubfiz(as_Register($dst$$reg),
13088           as_Register($src$$reg), lshift, width);
13089   %}
13090   ins_pipe(ialu_reg_shift);
13091 %}
13092 
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 
13096 // We can use ubfiz when masking by a positive number and then left shifting the result.
13097 // We know that the mask is positive because immI_bitmask guarantees it.
13098 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13099 %{
13100   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13101   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13102 
13103   ins_cost(INSN_COST);
13104   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13105   ins_encode %{
13106     int lshift = $lshift$$constant & 31;
13107     intptr_t mask = $mask$$constant;
13108     int width = exact_log2(mask+1);
13109     __ ubfizw(as_Register($dst$$reg),
13110           as_Register($src$$reg), lshift, width);
13111   %}
13112   ins_pipe(ialu_reg_shift);
13113 %}
13114 
13115 // This pattern is automatically generated from aarch64_ad.m4
13116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13117 
13118 // We can use ubfiz when masking by a positive number and then left shifting the result.
13119 // We know that the mask is positive because immL_bitmask guarantees it.
13120 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13121 %{
13122   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13123   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13124 
13125   ins_cost(INSN_COST);
13126   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13127   ins_encode %{
13128     int lshift = $lshift$$constant & 63;
13129     intptr_t mask = $mask$$constant;
13130     int width = exact_log2_long(mask+1);
13131     __ ubfiz(as_Register($dst$$reg),
13132           as_Register($src$$reg), lshift, width);
13133   %}
13134   ins_pipe(ialu_reg_shift);
13135 %}
13136 
13137 
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 
13141 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13142 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13143 %{
13144   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13145   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13146 
13147   ins_cost(INSN_COST);
13148   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13149   ins_encode %{
13150     int lshift = $lshift$$constant & 63;
13151     intptr_t mask = $mask$$constant;
13152     int width = exact_log2(mask+1);
13153     __ ubfiz(as_Register($dst$$reg),
13154              as_Register($src$$reg), lshift, width);
13155   %}
13156   ins_pipe(ialu_reg_shift);
13157 %}
13158 
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 
13162 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13163 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13164 %{
13165   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13166   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13167 
13168   ins_cost(INSN_COST);
13169   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13170   ins_encode %{
13171     int lshift = $lshift$$constant & 31;
13172     intptr_t mask = $mask$$constant;
13173     int width = exact_log2(mask+1);
13174     __ ubfiz(as_Register($dst$$reg),
13175              as_Register($src$$reg), lshift, width);
13176   %}
13177   ins_pipe(ialu_reg_shift);
13178 %}
13179 
13180 // This pattern is automatically generated from aarch64_ad.m4
13181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13182 
13183 // Can skip int2long conversions after AND with small bitmask
13184 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13185 %{
13186   match(Set dst (ConvI2L (AndI src msk)));
13187   ins_cost(INSN_COST);
13188   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13189   ins_encode %{
13190     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13191   %}
13192   ins_pipe(ialu_reg_shift);
13193 %}
13194 
13195 
13196 // Rotations
13197 // This pattern is automatically generated from aarch64_ad.m4
13198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13199 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13200 %{
13201   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13202   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13203 
13204   ins_cost(INSN_COST);
13205   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13206 
13207   ins_encode %{
13208     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13209             $rshift$$constant & 63);
13210   %}
13211   ins_pipe(ialu_reg_reg_extr);
13212 %}
13213 
13214 
13215 // This pattern is automatically generated from aarch64_ad.m4
13216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13217 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13218 %{
13219   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13220   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13221 
13222   ins_cost(INSN_COST);
13223   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13224 
13225   ins_encode %{
13226     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13227             $rshift$$constant & 31);
13228   %}
13229   ins_pipe(ialu_reg_reg_extr);
13230 %}
13231 
13232 
13233 // This pattern is automatically generated from aarch64_ad.m4
13234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13235 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13236 %{
13237   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13238   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13239 
13240   ins_cost(INSN_COST);
13241   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13242 
13243   ins_encode %{
13244     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13245             $rshift$$constant & 63);
13246   %}
13247   ins_pipe(ialu_reg_reg_extr);
13248 %}
13249 
13250 
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13254 %{
13255   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13256   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13257 
13258   ins_cost(INSN_COST);
13259   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13260 
13261   ins_encode %{
13262     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13263             $rshift$$constant & 31);
13264   %}
13265   ins_pipe(ialu_reg_reg_extr);
13266 %}
13267 
13268 
13269 // This pattern is automatically generated from aarch64_ad.m4
13270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13271 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13272 %{
13273   match(Set dst (RotateRight src shift));
13274 
13275   ins_cost(INSN_COST);
13276   format %{ "ror    $dst, $src, $shift" %}
13277 
13278   ins_encode %{
13279      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13280                $shift$$constant & 0x1f);
13281   %}
13282   ins_pipe(ialu_reg_reg_vshift);
13283 %}
13284 
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13288 %{
13289   match(Set dst (RotateRight src shift));
13290 
13291   ins_cost(INSN_COST);
13292   format %{ "ror    $dst, $src, $shift" %}
13293 
13294   ins_encode %{
13295      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13296                $shift$$constant & 0x3f);
13297   %}
13298   ins_pipe(ialu_reg_reg_vshift);
13299 %}
13300 
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13304 %{
13305   match(Set dst (RotateRight src shift));
13306 
13307   ins_cost(INSN_COST);
13308   format %{ "ror    $dst, $src, $shift" %}
13309 
13310   ins_encode %{
13311      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13312   %}
13313   ins_pipe(ialu_reg_reg_vshift);
13314 %}
13315 
13316 // This pattern is automatically generated from aarch64_ad.m4
13317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13318 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13319 %{
13320   match(Set dst (RotateRight src shift));
13321 
13322   ins_cost(INSN_COST);
13323   format %{ "ror    $dst, $src, $shift" %}
13324 
13325   ins_encode %{
13326      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13327   %}
13328   ins_pipe(ialu_reg_reg_vshift);
13329 %}
13330 
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13334 %{
13335   match(Set dst (RotateLeft src shift));
13336 
13337   ins_cost(INSN_COST);
13338   format %{ "rol    $dst, $src, $shift" %}
13339 
13340   ins_encode %{
13341      __ subw(rscratch1, zr, as_Register($shift$$reg));
13342      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13343   %}
13344   ins_pipe(ialu_reg_reg_vshift);
13345 %}
13346 
13347 // This pattern is automatically generated from aarch64_ad.m4
13348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13349 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13350 %{
13351   match(Set dst (RotateLeft src shift));
13352 
13353   ins_cost(INSN_COST);
13354   format %{ "rol    $dst, $src, $shift" %}
13355 
13356   ins_encode %{
13357      __ subw(rscratch1, zr, as_Register($shift$$reg));
13358      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13359   %}
13360   ins_pipe(ialu_reg_reg_vshift);
13361 %}
13362 
13363 
13364 // Add/subtract (extended)
13365 
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13369 %{
13370   match(Set dst (AddL src1 (ConvI2L src2)));
13371   ins_cost(INSN_COST);
13372   format %{ "add  $dst, $src1, $src2, sxtw" %}
13373 
13374    ins_encode %{
13375      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13376             as_Register($src2$$reg), ext::sxtw);
13377    %}
13378   ins_pipe(ialu_reg_reg);
13379 %}
13380 
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13384 %{
13385   match(Set dst (SubL src1 (ConvI2L src2)));
13386   ins_cost(INSN_COST);
13387   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13388 
13389    ins_encode %{
13390      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13391             as_Register($src2$$reg), ext::sxtw);
13392    %}
13393   ins_pipe(ialu_reg_reg);
13394 %}
13395 
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13399 %{
13400   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13401   ins_cost(INSN_COST);
13402   format %{ "add  $dst, $src1, $src2, sxth" %}
13403 
13404    ins_encode %{
13405      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13406             as_Register($src2$$reg), ext::sxth);
13407    %}
13408   ins_pipe(ialu_reg_reg);
13409 %}
13410 
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13414 %{
13415   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13416   ins_cost(INSN_COST);
13417   format %{ "add  $dst, $src1, $src2, sxtb" %}
13418 
13419    ins_encode %{
13420      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13421             as_Register($src2$$reg), ext::sxtb);
13422    %}
13423   ins_pipe(ialu_reg_reg);
13424 %}
13425 
13426 // This pattern is automatically generated from aarch64_ad.m4
13427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13428 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13429 %{
13430   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13431   ins_cost(INSN_COST);
13432   format %{ "add  $dst, $src1, $src2, uxtb" %}
13433 
13434    ins_encode %{
13435      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13436             as_Register($src2$$reg), ext::uxtb);
13437    %}
13438   ins_pipe(ialu_reg_reg);
13439 %}
13440 
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13444 %{
13445   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13446   ins_cost(INSN_COST);
13447   format %{ "add  $dst, $src1, $src2, sxth" %}
13448 
13449    ins_encode %{
13450      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13451             as_Register($src2$$reg), ext::sxth);
13452    %}
13453   ins_pipe(ialu_reg_reg);
13454 %}
13455 
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13459 %{
13460   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13461   ins_cost(INSN_COST);
13462   format %{ "add  $dst, $src1, $src2, sxtw" %}
13463 
13464    ins_encode %{
13465      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13466             as_Register($src2$$reg), ext::sxtw);
13467    %}
13468   ins_pipe(ialu_reg_reg);
13469 %}
13470 
13471 // This pattern is automatically generated from aarch64_ad.m4
13472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13473 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13474 %{
13475   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13476   ins_cost(INSN_COST);
13477   format %{ "add  $dst, $src1, $src2, sxtb" %}
13478 
13479    ins_encode %{
13480      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13481             as_Register($src2$$reg), ext::sxtb);
13482    %}
13483   ins_pipe(ialu_reg_reg);
13484 %}
13485 
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13489 %{
13490   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13491   ins_cost(INSN_COST);
13492   format %{ "add  $dst, $src1, $src2, uxtb" %}
13493 
13494    ins_encode %{
13495      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13496             as_Register($src2$$reg), ext::uxtb);
13497    %}
13498   ins_pipe(ialu_reg_reg);
13499 %}
13500 
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13504 %{
13505   match(Set dst (AddI src1 (AndI src2 mask)));
13506   ins_cost(INSN_COST);
13507   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13508 
13509    ins_encode %{
13510      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13511             as_Register($src2$$reg), ext::uxtb);
13512    %}
13513   ins_pipe(ialu_reg_reg);
13514 %}
13515 
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13519 %{
13520   match(Set dst (AddI src1 (AndI src2 mask)));
13521   ins_cost(INSN_COST);
13522   format %{ "addw  $dst, $src1, $src2, uxth" %}
13523 
13524    ins_encode %{
13525      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13526             as_Register($src2$$reg), ext::uxth);
13527    %}
13528   ins_pipe(ialu_reg_reg);
13529 %}
13530 
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13534 %{
13535   match(Set dst (AddL src1 (AndL src2 mask)));
13536   ins_cost(INSN_COST);
13537   format %{ "add  $dst, $src1, $src2, uxtb" %}
13538 
13539    ins_encode %{
13540      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13541             as_Register($src2$$reg), ext::uxtb);
13542    %}
13543   ins_pipe(ialu_reg_reg);
13544 %}
13545 
13546 // This pattern is automatically generated from aarch64_ad.m4
13547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13548 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13549 %{
13550   match(Set dst (AddL src1 (AndL src2 mask)));
13551   ins_cost(INSN_COST);
13552   format %{ "add  $dst, $src1, $src2, uxth" %}
13553 
13554    ins_encode %{
13555      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13556             as_Register($src2$$reg), ext::uxth);
13557    %}
13558   ins_pipe(ialu_reg_reg);
13559 %}
13560 
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13564 %{
13565   match(Set dst (AddL src1 (AndL src2 mask)));
13566   ins_cost(INSN_COST);
13567   format %{ "add  $dst, $src1, $src2, uxtw" %}
13568 
13569    ins_encode %{
13570      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13571             as_Register($src2$$reg), ext::uxtw);
13572    %}
13573   ins_pipe(ialu_reg_reg);
13574 %}
13575 
13576 // This pattern is automatically generated from aarch64_ad.m4
13577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13578 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13579 %{
13580   match(Set dst (SubI src1 (AndI src2 mask)));
13581   ins_cost(INSN_COST);
13582   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13583 
13584    ins_encode %{
13585      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13586             as_Register($src2$$reg), ext::uxtb);
13587    %}
13588   ins_pipe(ialu_reg_reg);
13589 %}
13590 
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13594 %{
13595   match(Set dst (SubI src1 (AndI src2 mask)));
13596   ins_cost(INSN_COST);
13597   format %{ "subw  $dst, $src1, $src2, uxth" %}
13598 
13599    ins_encode %{
13600      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13601             as_Register($src2$$reg), ext::uxth);
13602    %}
13603   ins_pipe(ialu_reg_reg);
13604 %}
13605 
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13609 %{
13610   match(Set dst (SubL src1 (AndL src2 mask)));
13611   ins_cost(INSN_COST);
13612   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13613 
13614    ins_encode %{
13615      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13616             as_Register($src2$$reg), ext::uxtb);
13617    %}
13618   ins_pipe(ialu_reg_reg);
13619 %}
13620 
13621 // This pattern is automatically generated from aarch64_ad.m4
13622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13623 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13624 %{
13625   match(Set dst (SubL src1 (AndL src2 mask)));
13626   ins_cost(INSN_COST);
13627   format %{ "sub  $dst, $src1, $src2, uxth" %}
13628 
13629    ins_encode %{
13630      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13631             as_Register($src2$$reg), ext::uxth);
13632    %}
13633   ins_pipe(ialu_reg_reg);
13634 %}
13635 
13636 // This pattern is automatically generated from aarch64_ad.m4
13637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13638 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13639 %{
13640   match(Set dst (SubL src1 (AndL src2 mask)));
13641   ins_cost(INSN_COST);
13642   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13643 
13644    ins_encode %{
13645      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13646             as_Register($src2$$reg), ext::uxtw);
13647    %}
13648   ins_pipe(ialu_reg_reg);
13649 %}
13650 
13651 
13652 // This pattern is automatically generated from aarch64_ad.m4
13653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13654 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13655 %{
13656   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13657   ins_cost(1.9 * INSN_COST);
13658   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13659 
13660    ins_encode %{
13661      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13662             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13663    %}
13664   ins_pipe(ialu_reg_reg_shift);
13665 %}
13666 
13667 // This pattern is automatically generated from aarch64_ad.m4
13668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13669 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13670 %{
13671   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13672   ins_cost(1.9 * INSN_COST);
13673   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13674 
13675    ins_encode %{
13676      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13677             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13678    %}
13679   ins_pipe(ialu_reg_reg_shift);
13680 %}
13681 
13682 // This pattern is automatically generated from aarch64_ad.m4
13683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13684 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13685 %{
13686   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13687   ins_cost(1.9 * INSN_COST);
13688   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13689 
13690    ins_encode %{
13691      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13692             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13693    %}
13694   ins_pipe(ialu_reg_reg_shift);
13695 %}
13696 
13697 // This pattern is automatically generated from aarch64_ad.m4
13698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13699 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13700 %{
13701   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13702   ins_cost(1.9 * INSN_COST);
13703   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13704 
13705    ins_encode %{
13706      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13707             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13708    %}
13709   ins_pipe(ialu_reg_reg_shift);
13710 %}
13711 
13712 // This pattern is automatically generated from aarch64_ad.m4
13713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13714 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13715 %{
13716   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13717   ins_cost(1.9 * INSN_COST);
13718   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13719 
13720    ins_encode %{
13721      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13722             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13723    %}
13724   ins_pipe(ialu_reg_reg_shift);
13725 %}
13726 
13727 // This pattern is automatically generated from aarch64_ad.m4
13728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13729 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13730 %{
13731   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13732   ins_cost(1.9 * INSN_COST);
13733   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13734 
13735    ins_encode %{
13736      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13737             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13738    %}
13739   ins_pipe(ialu_reg_reg_shift);
13740 %}
13741 
13742 // This pattern is automatically generated from aarch64_ad.m4
13743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13744 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13745 %{
13746   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13747   ins_cost(1.9 * INSN_COST);
13748   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13749 
13750    ins_encode %{
13751      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13752             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13753    %}
13754   ins_pipe(ialu_reg_reg_shift);
13755 %}
13756 
13757 // This pattern is automatically generated from aarch64_ad.m4
13758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13759 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13760 %{
13761   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13762   ins_cost(1.9 * INSN_COST);
13763   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13764 
13765    ins_encode %{
13766      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13767             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13768    %}
13769   ins_pipe(ialu_reg_reg_shift);
13770 %}
13771 
13772 // This pattern is automatically generated from aarch64_ad.m4
13773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13774 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13775 %{
13776   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13777   ins_cost(1.9 * INSN_COST);
13778   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13779 
13780    ins_encode %{
13781      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13782             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13783    %}
13784   ins_pipe(ialu_reg_reg_shift);
13785 %}
13786 
13787 // This pattern is automatically generated from aarch64_ad.m4
13788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13789 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13790 %{
13791   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13792   ins_cost(1.9 * INSN_COST);
13793   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13794 
13795    ins_encode %{
13796      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13797             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13798    %}
13799   ins_pipe(ialu_reg_reg_shift);
13800 %}
13801 
13802 // This pattern is automatically generated from aarch64_ad.m4
13803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13804 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13805 %{
13806   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13807   ins_cost(1.9 * INSN_COST);
13808   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13809 
13810    ins_encode %{
13811      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13812             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13813    %}
13814   ins_pipe(ialu_reg_reg_shift);
13815 %}
13816 
13817 // This pattern is automatically generated from aarch64_ad.m4
13818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13819 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13820 %{
13821   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13822   ins_cost(1.9 * INSN_COST);
13823   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13824 
13825    ins_encode %{
13826      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13827             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13828    %}
13829   ins_pipe(ialu_reg_reg_shift);
13830 %}
13831 
13832 // This pattern is automatically generated from aarch64_ad.m4
13833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13834 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13835 %{
13836   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13837   ins_cost(1.9 * INSN_COST);
13838   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13839 
13840    ins_encode %{
13841      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13842             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13843    %}
13844   ins_pipe(ialu_reg_reg_shift);
13845 %}
13846 
13847 // This pattern is automatically generated from aarch64_ad.m4
13848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13849 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13850 %{
13851   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13852   ins_cost(1.9 * INSN_COST);
13853   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13854 
13855    ins_encode %{
13856      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13857             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13858    %}
13859   ins_pipe(ialu_reg_reg_shift);
13860 %}
13861 
13862 // This pattern is automatically generated from aarch64_ad.m4
13863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13864 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13865 %{
13866   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13867   ins_cost(1.9 * INSN_COST);
13868   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13869 
13870    ins_encode %{
13871      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13872             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13873    %}
13874   ins_pipe(ialu_reg_reg_shift);
13875 %}
13876 
13877 // This pattern is automatically generated from aarch64_ad.m4
13878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13879 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13880 %{
13881   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13882   ins_cost(1.9 * INSN_COST);
13883   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13884 
13885    ins_encode %{
13886      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13887             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13888    %}
13889   ins_pipe(ialu_reg_reg_shift);
13890 %}
13891 
13892 // This pattern is automatically generated from aarch64_ad.m4
13893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13894 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13895 %{
13896   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13897   ins_cost(1.9 * INSN_COST);
13898   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13899 
13900    ins_encode %{
13901      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13902             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13903    %}
13904   ins_pipe(ialu_reg_reg_shift);
13905 %}
13906 
13907 // This pattern is automatically generated from aarch64_ad.m4
13908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13909 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13910 %{
13911   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13912   ins_cost(1.9 * INSN_COST);
13913   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13914 
13915    ins_encode %{
13916      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13917             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13918    %}
13919   ins_pipe(ialu_reg_reg_shift);
13920 %}
13921 
13922 // This pattern is automatically generated from aarch64_ad.m4
13923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13924 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13925 %{
13926   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13927   ins_cost(1.9 * INSN_COST);
13928   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13929 
13930    ins_encode %{
13931      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13932             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13933    %}
13934   ins_pipe(ialu_reg_reg_shift);
13935 %}
13936 
13937 // This pattern is automatically generated from aarch64_ad.m4
13938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13939 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13940 %{
13941   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13942   ins_cost(1.9 * INSN_COST);
13943   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13944 
13945    ins_encode %{
13946      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13947             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13948    %}
13949   ins_pipe(ialu_reg_reg_shift);
13950 %}
13951 
13952 // This pattern is automatically generated from aarch64_ad.m4
13953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13954 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13955 %{
13956   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13957   ins_cost(1.9 * INSN_COST);
13958   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13959 
13960    ins_encode %{
13961      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13962             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13963    %}
13964   ins_pipe(ialu_reg_reg_shift);
13965 %}
13966 
13967 // This pattern is automatically generated from aarch64_ad.m4
13968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13969 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13970 %{
13971   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13972   ins_cost(1.9 * INSN_COST);
13973   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13974 
13975    ins_encode %{
13976      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13977             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13978    %}
13979   ins_pipe(ialu_reg_reg_shift);
13980 %}
13981 
13982 
13983 
13984 // END This section of the file is automatically generated. Do not edit --------------
13985 
13986 
13987 // ============================================================================
13988 // Floating Point Arithmetic Instructions
13989 
13990 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13991   match(Set dst (AddF src1 src2));
13992 
13993   ins_cost(INSN_COST * 5);
13994   format %{ "fadds   $dst, $src1, $src2" %}
13995 
13996   ins_encode %{
13997     __ fadds(as_FloatRegister($dst$$reg),
13998              as_FloatRegister($src1$$reg),
13999              as_FloatRegister($src2$$reg));
14000   %}
14001 
14002   ins_pipe(fp_dop_reg_reg_s);
14003 %}
14004 
14005 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14006   match(Set dst (AddD src1 src2));
14007 
14008   ins_cost(INSN_COST * 5);
14009   format %{ "faddd   $dst, $src1, $src2" %}
14010 
14011   ins_encode %{
14012     __ faddd(as_FloatRegister($dst$$reg),
14013              as_FloatRegister($src1$$reg),
14014              as_FloatRegister($src2$$reg));
14015   %}
14016 
14017   ins_pipe(fp_dop_reg_reg_d);
14018 %}
14019 
14020 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14021   match(Set dst (SubF src1 src2));
14022 
14023   ins_cost(INSN_COST * 5);
14024   format %{ "fsubs   $dst, $src1, $src2" %}
14025 
14026   ins_encode %{
14027     __ fsubs(as_FloatRegister($dst$$reg),
14028              as_FloatRegister($src1$$reg),
14029              as_FloatRegister($src2$$reg));
14030   %}
14031 
14032   ins_pipe(fp_dop_reg_reg_s);
14033 %}
14034 
14035 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14036   match(Set dst (SubD src1 src2));
14037 
14038   ins_cost(INSN_COST * 5);
14039   format %{ "fsubd   $dst, $src1, $src2" %}
14040 
14041   ins_encode %{
14042     __ fsubd(as_FloatRegister($dst$$reg),
14043              as_FloatRegister($src1$$reg),
14044              as_FloatRegister($src2$$reg));
14045   %}
14046 
14047   ins_pipe(fp_dop_reg_reg_d);
14048 %}
14049 
14050 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14051   match(Set dst (MulF src1 src2));
14052 
14053   ins_cost(INSN_COST * 6);
14054   format %{ "fmuls   $dst, $src1, $src2" %}
14055 
14056   ins_encode %{
14057     __ fmuls(as_FloatRegister($dst$$reg),
14058              as_FloatRegister($src1$$reg),
14059              as_FloatRegister($src2$$reg));
14060   %}
14061 
14062   ins_pipe(fp_dop_reg_reg_s);
14063 %}
14064 
14065 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14066   match(Set dst (MulD src1 src2));
14067 
14068   ins_cost(INSN_COST * 6);
14069   format %{ "fmuld   $dst, $src1, $src2" %}
14070 
14071   ins_encode %{
14072     __ fmuld(as_FloatRegister($dst$$reg),
14073              as_FloatRegister($src1$$reg),
14074              as_FloatRegister($src2$$reg));
14075   %}
14076 
14077   ins_pipe(fp_dop_reg_reg_d);
14078 %}
14079 
14080 // src1 * src2 + src3
14081 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14082   predicate(UseFMA);
14083   match(Set dst (FmaF src3 (Binary src1 src2)));
14084 
14085   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14086 
14087   ins_encode %{
14088     __ fmadds(as_FloatRegister($dst$$reg),
14089              as_FloatRegister($src1$$reg),
14090              as_FloatRegister($src2$$reg),
14091              as_FloatRegister($src3$$reg));
14092   %}
14093 
14094   ins_pipe(pipe_class_default);
14095 %}
14096 
14097 // src1 * src2 + src3
14098 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14099   predicate(UseFMA);
14100   match(Set dst (FmaD src3 (Binary src1 src2)));
14101 
14102   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14103 
14104   ins_encode %{
14105     __ fmaddd(as_FloatRegister($dst$$reg),
14106              as_FloatRegister($src1$$reg),
14107              as_FloatRegister($src2$$reg),
14108              as_FloatRegister($src3$$reg));
14109   %}
14110 
14111   ins_pipe(pipe_class_default);
14112 %}
14113 
14114 // -src1 * src2 + src3
14115 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14116   predicate(UseFMA);
14117   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14118   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14119 
14120   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14121 
14122   ins_encode %{
14123     __ fmsubs(as_FloatRegister($dst$$reg),
14124               as_FloatRegister($src1$$reg),
14125               as_FloatRegister($src2$$reg),
14126               as_FloatRegister($src3$$reg));
14127   %}
14128 
14129   ins_pipe(pipe_class_default);
14130 %}
14131 
14132 // -src1 * src2 + src3
14133 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14134   predicate(UseFMA);
14135   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14136   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14137 
14138   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14139 
14140   ins_encode %{
14141     __ fmsubd(as_FloatRegister($dst$$reg),
14142               as_FloatRegister($src1$$reg),
14143               as_FloatRegister($src2$$reg),
14144               as_FloatRegister($src3$$reg));
14145   %}
14146 
14147   ins_pipe(pipe_class_default);
14148 %}
14149 
14150 // -src1 * src2 - src3
14151 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14152   predicate(UseFMA);
14153   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14154   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14155 
14156   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14157 
14158   ins_encode %{
14159     __ fnmadds(as_FloatRegister($dst$$reg),
14160                as_FloatRegister($src1$$reg),
14161                as_FloatRegister($src2$$reg),
14162                as_FloatRegister($src3$$reg));
14163   %}
14164 
14165   ins_pipe(pipe_class_default);
14166 %}
14167 
14168 // -src1 * src2 - src3
14169 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14170   predicate(UseFMA);
14171   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14172   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14173 
14174   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14175 
14176   ins_encode %{
14177     __ fnmaddd(as_FloatRegister($dst$$reg),
14178                as_FloatRegister($src1$$reg),
14179                as_FloatRegister($src2$$reg),
14180                as_FloatRegister($src3$$reg));
14181   %}
14182 
14183   ins_pipe(pipe_class_default);
14184 %}
14185 
14186 // src1 * src2 - src3
14187 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14188   predicate(UseFMA);
14189   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14190 
14191   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14192 
14193   ins_encode %{
14194     __ fnmsubs(as_FloatRegister($dst$$reg),
14195                as_FloatRegister($src1$$reg),
14196                as_FloatRegister($src2$$reg),
14197                as_FloatRegister($src3$$reg));
14198   %}
14199 
14200   ins_pipe(pipe_class_default);
14201 %}
14202 
14203 // src1 * src2 - src3
14204 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14205   predicate(UseFMA);
14206   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14207 
14208   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14209 
14210   ins_encode %{
14211   // n.b. insn name should be fnmsubd
14212     __ fnmsub(as_FloatRegister($dst$$reg),
14213               as_FloatRegister($src1$$reg),
14214               as_FloatRegister($src2$$reg),
14215               as_FloatRegister($src3$$reg));
14216   %}
14217 
14218   ins_pipe(pipe_class_default);
14219 %}
14220 
14221 
14222 // Math.max(FF)F
14223 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14224   match(Set dst (MaxF src1 src2));
14225 
14226   format %{ "fmaxs   $dst, $src1, $src2" %}
14227   ins_encode %{
14228     __ fmaxs(as_FloatRegister($dst$$reg),
14229              as_FloatRegister($src1$$reg),
14230              as_FloatRegister($src2$$reg));
14231   %}
14232 
14233   ins_pipe(fp_dop_reg_reg_s);
14234 %}
14235 
14236 // Math.min(FF)F
14237 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14238   match(Set dst (MinF src1 src2));
14239 
14240   format %{ "fmins   $dst, $src1, $src2" %}
14241   ins_encode %{
14242     __ fmins(as_FloatRegister($dst$$reg),
14243              as_FloatRegister($src1$$reg),
14244              as_FloatRegister($src2$$reg));
14245   %}
14246 
14247   ins_pipe(fp_dop_reg_reg_s);
14248 %}
14249 
14250 // Math.max(DD)D
14251 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14252   match(Set dst (MaxD src1 src2));
14253 
14254   format %{ "fmaxd   $dst, $src1, $src2" %}
14255   ins_encode %{
14256     __ fmaxd(as_FloatRegister($dst$$reg),
14257              as_FloatRegister($src1$$reg),
14258              as_FloatRegister($src2$$reg));
14259   %}
14260 
14261   ins_pipe(fp_dop_reg_reg_d);
14262 %}
14263 
14264 // Math.min(DD)D
14265 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14266   match(Set dst (MinD src1 src2));
14267 
14268   format %{ "fmind   $dst, $src1, $src2" %}
14269   ins_encode %{
14270     __ fmind(as_FloatRegister($dst$$reg),
14271              as_FloatRegister($src1$$reg),
14272              as_FloatRegister($src2$$reg));
14273   %}
14274 
14275   ins_pipe(fp_dop_reg_reg_d);
14276 %}
14277 
14278 
14279 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14280   match(Set dst (DivF src1  src2));
14281 
14282   ins_cost(INSN_COST * 18);
14283   format %{ "fdivs   $dst, $src1, $src2" %}
14284 
14285   ins_encode %{
14286     __ fdivs(as_FloatRegister($dst$$reg),
14287              as_FloatRegister($src1$$reg),
14288              as_FloatRegister($src2$$reg));
14289   %}
14290 
14291   ins_pipe(fp_div_s);
14292 %}
14293 
14294 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14295   match(Set dst (DivD src1  src2));
14296 
14297   ins_cost(INSN_COST * 32);
14298   format %{ "fdivd   $dst, $src1, $src2" %}
14299 
14300   ins_encode %{
14301     __ fdivd(as_FloatRegister($dst$$reg),
14302              as_FloatRegister($src1$$reg),
14303              as_FloatRegister($src2$$reg));
14304   %}
14305 
14306   ins_pipe(fp_div_d);
14307 %}
14308 
14309 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14310   match(Set dst (NegF src));
14311 
14312   ins_cost(INSN_COST * 3);
14313   format %{ "fneg   $dst, $src" %}
14314 
14315   ins_encode %{
14316     __ fnegs(as_FloatRegister($dst$$reg),
14317              as_FloatRegister($src$$reg));
14318   %}
14319 
14320   ins_pipe(fp_uop_s);
14321 %}
14322 
14323 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14324   match(Set dst (NegD src));
14325 
14326   ins_cost(INSN_COST * 3);
14327   format %{ "fnegd   $dst, $src" %}
14328 
14329   ins_encode %{
14330     __ fnegd(as_FloatRegister($dst$$reg),
14331              as_FloatRegister($src$$reg));
14332   %}
14333 
14334   ins_pipe(fp_uop_d);
14335 %}
14336 
14337 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14338 %{
14339   match(Set dst (AbsI src));
14340 
14341   effect(KILL cr);
14342   ins_cost(INSN_COST * 2);
14343   format %{ "cmpw  $src, zr\n\t"
14344             "cnegw $dst, $src, Assembler::LT\t# int abs"
14345   %}
14346 
14347   ins_encode %{
14348     __ cmpw(as_Register($src$$reg), zr);
14349     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14350   %}
14351   ins_pipe(pipe_class_default);
14352 %}
14353 
14354 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14355 %{
14356   match(Set dst (AbsL src));
14357 
14358   effect(KILL cr);
14359   ins_cost(INSN_COST * 2);
14360   format %{ "cmp  $src, zr\n\t"
14361             "cneg $dst, $src, Assembler::LT\t# long abs"
14362   %}
14363 
14364   ins_encode %{
14365     __ cmp(as_Register($src$$reg), zr);
14366     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14367   %}
14368   ins_pipe(pipe_class_default);
14369 %}
14370 
14371 instruct absF_reg(vRegF dst, vRegF src) %{
14372   match(Set dst (AbsF src));
14373 
14374   ins_cost(INSN_COST * 3);
14375   format %{ "fabss   $dst, $src" %}
14376   ins_encode %{
14377     __ fabss(as_FloatRegister($dst$$reg),
14378              as_FloatRegister($src$$reg));
14379   %}
14380 
14381   ins_pipe(fp_uop_s);
14382 %}
14383 
14384 instruct absD_reg(vRegD dst, vRegD src) %{
14385   match(Set dst (AbsD src));
14386 
14387   ins_cost(INSN_COST * 3);
14388   format %{ "fabsd   $dst, $src" %}
14389   ins_encode %{
14390     __ fabsd(as_FloatRegister($dst$$reg),
14391              as_FloatRegister($src$$reg));
14392   %}
14393 
14394   ins_pipe(fp_uop_d);
14395 %}
14396 
14397 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14398   match(Set dst (AbsF (SubF src1 src2)));
14399 
14400   ins_cost(INSN_COST * 3);
14401   format %{ "fabds   $dst, $src1, $src2" %}
14402   ins_encode %{
14403     __ fabds(as_FloatRegister($dst$$reg),
14404              as_FloatRegister($src1$$reg),
14405              as_FloatRegister($src2$$reg));
14406   %}
14407 
14408   ins_pipe(fp_uop_s);
14409 %}
14410 
14411 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14412   match(Set dst (AbsD (SubD src1 src2)));
14413 
14414   ins_cost(INSN_COST * 3);
14415   format %{ "fabdd   $dst, $src1, $src2" %}
14416   ins_encode %{
14417     __ fabdd(as_FloatRegister($dst$$reg),
14418              as_FloatRegister($src1$$reg),
14419              as_FloatRegister($src2$$reg));
14420   %}
14421 
14422   ins_pipe(fp_uop_d);
14423 %}
14424 
14425 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14426   match(Set dst (SqrtD src));
14427 
14428   ins_cost(INSN_COST * 50);
14429   format %{ "fsqrtd  $dst, $src" %}
14430   ins_encode %{
14431     __ fsqrtd(as_FloatRegister($dst$$reg),
14432              as_FloatRegister($src$$reg));
14433   %}
14434 
14435   ins_pipe(fp_div_s);
14436 %}
14437 
14438 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14439   match(Set dst (SqrtF src));
14440 
14441   ins_cost(INSN_COST * 50);
14442   format %{ "fsqrts  $dst, $src" %}
14443   ins_encode %{
14444     __ fsqrts(as_FloatRegister($dst$$reg),
14445              as_FloatRegister($src$$reg));
14446   %}
14447 
14448   ins_pipe(fp_div_d);
14449 %}
14450 
14451 // Math.rint, floor, ceil
14452 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14453   match(Set dst (RoundDoubleMode src rmode));
14454   format %{ "frint  $dst, $src, $rmode" %}
14455   ins_encode %{
14456     switch ($rmode$$constant) {
14457       case RoundDoubleModeNode::rmode_rint:
14458         __ frintnd(as_FloatRegister($dst$$reg),
14459                    as_FloatRegister($src$$reg));
14460         break;
14461       case RoundDoubleModeNode::rmode_floor:
14462         __ frintmd(as_FloatRegister($dst$$reg),
14463                    as_FloatRegister($src$$reg));
14464         break;
14465       case RoundDoubleModeNode::rmode_ceil:
14466         __ frintpd(as_FloatRegister($dst$$reg),
14467                    as_FloatRegister($src$$reg));
14468         break;
14469     }
14470   %}
14471   ins_pipe(fp_uop_d);
14472 %}
14473 
14474 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14475   match(Set dst (CopySignD src1 (Binary src2 zero)));
14476   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14477   format %{ "CopySignD  $dst $src1 $src2" %}
14478   ins_encode %{
14479     FloatRegister dst = as_FloatRegister($dst$$reg),
14480                   src1 = as_FloatRegister($src1$$reg),
14481                   src2 = as_FloatRegister($src2$$reg),
14482                   zero = as_FloatRegister($zero$$reg);
14483     __ fnegd(dst, zero);
14484     __ bsl(dst, __ T8B, src2, src1);
14485   %}
14486   ins_pipe(fp_uop_d);
14487 %}
14488 
14489 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14490   match(Set dst (CopySignF src1 src2));
14491   effect(TEMP_DEF dst, USE src1, USE src2);
14492   format %{ "CopySignF  $dst $src1 $src2" %}
14493   ins_encode %{
14494     FloatRegister dst = as_FloatRegister($dst$$reg),
14495                   src1 = as_FloatRegister($src1$$reg),
14496                   src2 = as_FloatRegister($src2$$reg);
14497     __ movi(dst, __ T2S, 0x80, 24);
14498     __ bsl(dst, __ T8B, src2, src1);
14499   %}
14500   ins_pipe(fp_uop_d);
14501 %}
14502 
14503 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14504   match(Set dst (SignumD src (Binary zero one)));
14505   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14506   format %{ "signumD  $dst, $src" %}
14507   ins_encode %{
14508     FloatRegister src = as_FloatRegister($src$$reg),
14509                   dst = as_FloatRegister($dst$$reg),
14510                   zero = as_FloatRegister($zero$$reg),
14511                   one = as_FloatRegister($one$$reg);
14512     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14513     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14514     // Bit selection instruction gets bit from "one" for each enabled bit in
14515     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14516     // NaN the whole "src" will be copied because "dst" is zero. For all other
14517     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14518     // from "src", and all other bits are copied from 1.0.
14519     __ bsl(dst, __ T8B, one, src);
14520   %}
14521   ins_pipe(fp_uop_d);
14522 %}
14523 
14524 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14525   match(Set dst (SignumF src (Binary zero one)));
14526   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14527   format %{ "signumF  $dst, $src" %}
14528   ins_encode %{
14529     FloatRegister src = as_FloatRegister($src$$reg),
14530                   dst = as_FloatRegister($dst$$reg),
14531                   zero = as_FloatRegister($zero$$reg),
14532                   one = as_FloatRegister($one$$reg);
14533     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14534     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14535     // Bit selection instruction gets bit from "one" for each enabled bit in
14536     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14537     // NaN the whole "src" will be copied because "dst" is zero. For all other
14538     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14539     // from "src", and all other bits are copied from 1.0.
14540     __ bsl(dst, __ T8B, one, src);
14541   %}
14542   ins_pipe(fp_uop_d);
14543 %}
14544 
14545 instruct onspinwait() %{
14546   match(OnSpinWait);
14547   ins_cost(INSN_COST);
14548 
14549   format %{ "onspinwait" %}
14550 
14551   ins_encode %{
14552     __ spin_wait();
14553   %}
14554   ins_pipe(pipe_class_empty);
14555 %}
14556 
14557 // ============================================================================
14558 // Logical Instructions
14559 
14560 // Integer Logical Instructions
14561 
14562 // And Instructions
14563 
14564 
14565 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14566   match(Set dst (AndI src1 src2));
14567 
14568   format %{ "andw  $dst, $src1, $src2\t# int" %}
14569 
14570   ins_cost(INSN_COST);
14571   ins_encode %{
14572     __ andw(as_Register($dst$$reg),
14573             as_Register($src1$$reg),
14574             as_Register($src2$$reg));
14575   %}
14576 
14577   ins_pipe(ialu_reg_reg);
14578 %}
14579 
14580 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14581   match(Set dst (AndI src1 src2));
14582 
14583   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14584 
14585   ins_cost(INSN_COST);
14586   ins_encode %{
14587     __ andw(as_Register($dst$$reg),
14588             as_Register($src1$$reg),
14589             (uint64_t)($src2$$constant));
14590   %}
14591 
14592   ins_pipe(ialu_reg_imm);
14593 %}
14594 
14595 // Or Instructions
14596 
14597 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14598   match(Set dst (OrI src1 src2));
14599 
14600   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14601 
14602   ins_cost(INSN_COST);
14603   ins_encode %{
14604     __ orrw(as_Register($dst$$reg),
14605             as_Register($src1$$reg),
14606             as_Register($src2$$reg));
14607   %}
14608 
14609   ins_pipe(ialu_reg_reg);
14610 %}
14611 
14612 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14613   match(Set dst (OrI src1 src2));
14614 
14615   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14616 
14617   ins_cost(INSN_COST);
14618   ins_encode %{
14619     __ orrw(as_Register($dst$$reg),
14620             as_Register($src1$$reg),
14621             (uint64_t)($src2$$constant));
14622   %}
14623 
14624   ins_pipe(ialu_reg_imm);
14625 %}
14626 
14627 // Xor Instructions
14628 
14629 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14630   match(Set dst (XorI src1 src2));
14631 
14632   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14633 
14634   ins_cost(INSN_COST);
14635   ins_encode %{
14636     __ eorw(as_Register($dst$$reg),
14637             as_Register($src1$$reg),
14638             as_Register($src2$$reg));
14639   %}
14640 
14641   ins_pipe(ialu_reg_reg);
14642 %}
14643 
14644 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14645   match(Set dst (XorI src1 src2));
14646 
14647   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14648 
14649   ins_cost(INSN_COST);
14650   ins_encode %{
14651     __ eorw(as_Register($dst$$reg),
14652             as_Register($src1$$reg),
14653             (uint64_t)($src2$$constant));
14654   %}
14655 
14656   ins_pipe(ialu_reg_imm);
14657 %}
14658 
14659 // Long Logical Instructions
14660 // TODO
14661 
14662 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14663   match(Set dst (AndL src1 src2));
14664 
14665   format %{ "and  $dst, $src1, $src2\t# int" %}
14666 
14667   ins_cost(INSN_COST);
14668   ins_encode %{
14669     __ andr(as_Register($dst$$reg),
14670             as_Register($src1$$reg),
14671             as_Register($src2$$reg));
14672   %}
14673 
14674   ins_pipe(ialu_reg_reg);
14675 %}
14676 
14677 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14678   match(Set dst (AndL src1 src2));
14679 
14680   format %{ "and  $dst, $src1, $src2\t# int" %}
14681 
14682   ins_cost(INSN_COST);
14683   ins_encode %{
14684     __ andr(as_Register($dst$$reg),
14685             as_Register($src1$$reg),
14686             (uint64_t)($src2$$constant));
14687   %}
14688 
14689   ins_pipe(ialu_reg_imm);
14690 %}
14691 
14692 // Or Instructions
14693 
14694 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14695   match(Set dst (OrL src1 src2));
14696 
14697   format %{ "orr  $dst, $src1, $src2\t# int" %}
14698 
14699   ins_cost(INSN_COST);
14700   ins_encode %{
14701     __ orr(as_Register($dst$$reg),
14702            as_Register($src1$$reg),
14703            as_Register($src2$$reg));
14704   %}
14705 
14706   ins_pipe(ialu_reg_reg);
14707 %}
14708 
14709 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14710   match(Set dst (OrL src1 src2));
14711 
14712   format %{ "orr  $dst, $src1, $src2\t# int" %}
14713 
14714   ins_cost(INSN_COST);
14715   ins_encode %{
14716     __ orr(as_Register($dst$$reg),
14717            as_Register($src1$$reg),
14718            (uint64_t)($src2$$constant));
14719   %}
14720 
14721   ins_pipe(ialu_reg_imm);
14722 %}
14723 
14724 // Xor Instructions
14725 
14726 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14727   match(Set dst (XorL src1 src2));
14728 
14729   format %{ "eor  $dst, $src1, $src2\t# int" %}
14730 
14731   ins_cost(INSN_COST);
14732   ins_encode %{
14733     __ eor(as_Register($dst$$reg),
14734            as_Register($src1$$reg),
14735            as_Register($src2$$reg));
14736   %}
14737 
14738   ins_pipe(ialu_reg_reg);
14739 %}
14740 
14741 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14742   match(Set dst (XorL src1 src2));
14743 
14744   ins_cost(INSN_COST);
14745   format %{ "eor  $dst, $src1, $src2\t# int" %}
14746 
14747   ins_encode %{
14748     __ eor(as_Register($dst$$reg),
14749            as_Register($src1$$reg),
14750            (uint64_t)($src2$$constant));
14751   %}
14752 
14753   ins_pipe(ialu_reg_imm);
14754 %}
14755 
14756 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14757 %{
14758   match(Set dst (ConvI2L src));
14759 
14760   ins_cost(INSN_COST);
14761   format %{ "sxtw  $dst, $src\t# i2l" %}
14762   ins_encode %{
14763     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14764   %}
14765   ins_pipe(ialu_reg_shift);
14766 %}
14767 
14768 // this pattern occurs in bigmath arithmetic
14769 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14770 %{
14771   match(Set dst (AndL (ConvI2L src) mask));
14772 
14773   ins_cost(INSN_COST);
14774   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14775   ins_encode %{
14776     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14777   %}
14778 
14779   ins_pipe(ialu_reg_shift);
14780 %}
14781 
14782 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14783   match(Set dst (ConvL2I src));
14784 
14785   ins_cost(INSN_COST);
14786   format %{ "movw  $dst, $src \t// l2i" %}
14787 
14788   ins_encode %{
14789     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14790   %}
14791 
14792   ins_pipe(ialu_reg);
14793 %}
14794 
14795 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14796 %{
14797   match(Set dst (Conv2B src));
14798   effect(KILL cr);
14799 
14800   format %{
14801     "cmpw $src, zr\n\t"
14802     "cset $dst, ne"
14803   %}
14804 
14805   ins_encode %{
14806     __ cmpw(as_Register($src$$reg), zr);
14807     __ cset(as_Register($dst$$reg), Assembler::NE);
14808   %}
14809 
14810   ins_pipe(ialu_reg);
14811 %}
14812 
14813 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14814 %{
14815   match(Set dst (Conv2B src));
14816   effect(KILL cr);
14817 
14818   format %{
14819     "cmp  $src, zr\n\t"
14820     "cset $dst, ne"
14821   %}
14822 
14823   ins_encode %{
14824     __ cmp(as_Register($src$$reg), zr);
14825     __ cset(as_Register($dst$$reg), Assembler::NE);
14826   %}
14827 
14828   ins_pipe(ialu_reg);
14829 %}
14830 
14831 instruct convD2F_reg(vRegF dst, vRegD src) %{
14832   match(Set dst (ConvD2F src));
14833 
14834   ins_cost(INSN_COST * 5);
14835   format %{ "fcvtd  $dst, $src \t// d2f" %}
14836 
14837   ins_encode %{
14838     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14839   %}
14840 
14841   ins_pipe(fp_d2f);
14842 %}
14843 
14844 instruct convF2D_reg(vRegD dst, vRegF src) %{
14845   match(Set dst (ConvF2D src));
14846 
14847   ins_cost(INSN_COST * 5);
14848   format %{ "fcvts  $dst, $src \t// f2d" %}
14849 
14850   ins_encode %{
14851     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14852   %}
14853 
14854   ins_pipe(fp_f2d);
14855 %}
14856 
14857 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14858   match(Set dst (ConvF2I src));
14859 
14860   ins_cost(INSN_COST * 5);
14861   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14862 
14863   ins_encode %{
14864     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14865   %}
14866 
14867   ins_pipe(fp_f2i);
14868 %}
14869 
14870 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14871   match(Set dst (ConvF2L src));
14872 
14873   ins_cost(INSN_COST * 5);
14874   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14875 
14876   ins_encode %{
14877     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14878   %}
14879 
14880   ins_pipe(fp_f2l);
14881 %}
14882 
14883 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14884   match(Set dst (ConvI2F src));
14885 
14886   ins_cost(INSN_COST * 5);
14887   format %{ "scvtfws  $dst, $src \t// i2f" %}
14888 
14889   ins_encode %{
14890     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14891   %}
14892 
14893   ins_pipe(fp_i2f);
14894 %}
14895 
14896 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14897   match(Set dst (ConvL2F src));
14898 
14899   ins_cost(INSN_COST * 5);
14900   format %{ "scvtfs  $dst, $src \t// l2f" %}
14901 
14902   ins_encode %{
14903     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14904   %}
14905 
14906   ins_pipe(fp_l2f);
14907 %}
14908 
14909 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14910   match(Set dst (ConvD2I src));
14911 
14912   ins_cost(INSN_COST * 5);
14913   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14914 
14915   ins_encode %{
14916     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14917   %}
14918 
14919   ins_pipe(fp_d2i);
14920 %}
14921 
14922 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14923   match(Set dst (ConvD2L src));
14924 
14925   ins_cost(INSN_COST * 5);
14926   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14927 
14928   ins_encode %{
14929     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14930   %}
14931 
14932   ins_pipe(fp_d2l);
14933 %}
14934 
14935 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14936   match(Set dst (ConvI2D src));
14937 
14938   ins_cost(INSN_COST * 5);
14939   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14940 
14941   ins_encode %{
14942     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14943   %}
14944 
14945   ins_pipe(fp_i2d);
14946 %}
14947 
14948 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14949   match(Set dst (ConvL2D src));
14950 
14951   ins_cost(INSN_COST * 5);
14952   format %{ "scvtfd  $dst, $src \t// l2d" %}
14953 
14954   ins_encode %{
14955     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14956   %}
14957 
14958   ins_pipe(fp_l2d);
14959 %}
14960 
14961 // stack <-> reg and reg <-> reg shuffles with no conversion
14962 
14963 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14964 
14965   match(Set dst (MoveF2I src));
14966 
14967   effect(DEF dst, USE src);
14968 
14969   ins_cost(4 * INSN_COST);
14970 
14971   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14972 
14973   ins_encode %{
14974     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14975   %}
14976 
14977   ins_pipe(iload_reg_reg);
14978 
14979 %}
14980 
14981 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14982 
14983   match(Set dst (MoveI2F src));
14984 
14985   effect(DEF dst, USE src);
14986 
14987   ins_cost(4 * INSN_COST);
14988 
14989   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14990 
14991   ins_encode %{
14992     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14993   %}
14994 
14995   ins_pipe(pipe_class_memory);
14996 
14997 %}
14998 
14999 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15000 
15001   match(Set dst (MoveD2L src));
15002 
15003   effect(DEF dst, USE src);
15004 
15005   ins_cost(4 * INSN_COST);
15006 
15007   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15008 
15009   ins_encode %{
15010     __ ldr($dst$$Register, Address(sp, $src$$disp));
15011   %}
15012 
15013   ins_pipe(iload_reg_reg);
15014 
15015 %}
15016 
15017 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15018 
15019   match(Set dst (MoveL2D src));
15020 
15021   effect(DEF dst, USE src);
15022 
15023   ins_cost(4 * INSN_COST);
15024 
15025   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15026 
15027   ins_encode %{
15028     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15029   %}
15030 
15031   ins_pipe(pipe_class_memory);
15032 
15033 %}
15034 
15035 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15036 
15037   match(Set dst (MoveF2I src));
15038 
15039   effect(DEF dst, USE src);
15040 
15041   ins_cost(INSN_COST);
15042 
15043   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15044 
15045   ins_encode %{
15046     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15047   %}
15048 
15049   ins_pipe(pipe_class_memory);
15050 
15051 %}
15052 
15053 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15054 
15055   match(Set dst (MoveI2F src));
15056 
15057   effect(DEF dst, USE src);
15058 
15059   ins_cost(INSN_COST);
15060 
15061   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15062 
15063   ins_encode %{
15064     __ strw($src$$Register, Address(sp, $dst$$disp));
15065   %}
15066 
15067   ins_pipe(istore_reg_reg);
15068 
15069 %}
15070 
15071 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15072 
15073   match(Set dst (MoveD2L src));
15074 
15075   effect(DEF dst, USE src);
15076 
15077   ins_cost(INSN_COST);
15078 
15079   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15080 
15081   ins_encode %{
15082     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15083   %}
15084 
15085   ins_pipe(pipe_class_memory);
15086 
15087 %}
15088 
15089 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15090 
15091   match(Set dst (MoveL2D src));
15092 
15093   effect(DEF dst, USE src);
15094 
15095   ins_cost(INSN_COST);
15096 
15097   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15098 
15099   ins_encode %{
15100     __ str($src$$Register, Address(sp, $dst$$disp));
15101   %}
15102 
15103   ins_pipe(istore_reg_reg);
15104 
15105 %}
15106 
15107 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15108 
15109   match(Set dst (MoveF2I src));
15110 
15111   effect(DEF dst, USE src);
15112 
15113   ins_cost(INSN_COST);
15114 
15115   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15116 
15117   ins_encode %{
15118     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15119   %}
15120 
15121   ins_pipe(fp_f2i);
15122 
15123 %}
15124 
15125 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15126 
15127   match(Set dst (MoveI2F src));
15128 
15129   effect(DEF dst, USE src);
15130 
15131   ins_cost(INSN_COST);
15132 
15133   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15134 
15135   ins_encode %{
15136     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15137   %}
15138 
15139   ins_pipe(fp_i2f);
15140 
15141 %}
15142 
15143 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15144 
15145   match(Set dst (MoveD2L src));
15146 
15147   effect(DEF dst, USE src);
15148 
15149   ins_cost(INSN_COST);
15150 
15151   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15152 
15153   ins_encode %{
15154     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15155   %}
15156 
15157   ins_pipe(fp_d2l);
15158 
15159 %}
15160 
15161 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15162 
15163   match(Set dst (MoveL2D src));
15164 
15165   effect(DEF dst, USE src);
15166 
15167   ins_cost(INSN_COST);
15168 
15169   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15170 
15171   ins_encode %{
15172     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15173   %}
15174 
15175   ins_pipe(fp_l2d);
15176 
15177 %}
15178 
15179 // ============================================================================
15180 // clearing of an array
15181 
15182 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15183 %{
15184   match(Set dummy (ClearArray cnt base));
15185   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15186 
15187   ins_cost(4 * INSN_COST);
15188   format %{ "ClearArray $cnt, $base" %}
15189 
15190   ins_encode %{
15191     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15192     if (tpc == NULL) {
15193       ciEnv::current()->record_failure("CodeCache is full");
15194       return;
15195     }
15196   %}
15197 
15198   ins_pipe(pipe_class_memory);
15199 %}
15200 
15201 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15202 %{
15203   predicate((uint64_t)n->in(2)->get_long()
15204             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15205   match(Set dummy (ClearArray cnt base));
15206   effect(TEMP temp, USE_KILL base, KILL cr);
15207 
15208   ins_cost(4 * INSN_COST);
15209   format %{ "ClearArray $cnt, $base" %}
15210 
15211   ins_encode %{
15212     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15213     if (tpc == NULL) {
15214       ciEnv::current()->record_failure("CodeCache is full");
15215       return;
15216     }
15217   %}
15218 
15219   ins_pipe(pipe_class_memory);
15220 %}
15221 
15222 // ============================================================================
15223 // Overflow Math Instructions
15224 
15225 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15226 %{
15227   match(Set cr (OverflowAddI op1 op2));
15228 
15229   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15230   ins_cost(INSN_COST);
15231   ins_encode %{
15232     __ cmnw($op1$$Register, $op2$$Register);
15233   %}
15234 
15235   ins_pipe(icmp_reg_reg);
15236 %}
15237 
15238 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15239 %{
15240   match(Set cr (OverflowAddI op1 op2));
15241 
15242   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15243   ins_cost(INSN_COST);
15244   ins_encode %{
15245     __ cmnw($op1$$Register, $op2$$constant);
15246   %}
15247 
15248   ins_pipe(icmp_reg_imm);
15249 %}
15250 
15251 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15252 %{
15253   match(Set cr (OverflowAddL op1 op2));
15254 
15255   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15256   ins_cost(INSN_COST);
15257   ins_encode %{
15258     __ cmn($op1$$Register, $op2$$Register);
15259   %}
15260 
15261   ins_pipe(icmp_reg_reg);
15262 %}
15263 
15264 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15265 %{
15266   match(Set cr (OverflowAddL op1 op2));
15267 
15268   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15269   ins_cost(INSN_COST);
15270   ins_encode %{
15271     __ adds(zr, $op1$$Register, $op2$$constant);
15272   %}
15273 
15274   ins_pipe(icmp_reg_imm);
15275 %}
15276 
15277 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15278 %{
15279   match(Set cr (OverflowSubI op1 op2));
15280 
15281   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15282   ins_cost(INSN_COST);
15283   ins_encode %{
15284     __ cmpw($op1$$Register, $op2$$Register);
15285   %}
15286 
15287   ins_pipe(icmp_reg_reg);
15288 %}
15289 
15290 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15291 %{
15292   match(Set cr (OverflowSubI op1 op2));
15293 
15294   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15295   ins_cost(INSN_COST);
15296   ins_encode %{
15297     __ cmpw($op1$$Register, $op2$$constant);
15298   %}
15299 
15300   ins_pipe(icmp_reg_imm);
15301 %}
15302 
15303 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15304 %{
15305   match(Set cr (OverflowSubL op1 op2));
15306 
15307   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15308   ins_cost(INSN_COST);
15309   ins_encode %{
15310     __ cmp($op1$$Register, $op2$$Register);
15311   %}
15312 
15313   ins_pipe(icmp_reg_reg);
15314 %}
15315 
15316 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15317 %{
15318   match(Set cr (OverflowSubL op1 op2));
15319 
15320   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15321   ins_cost(INSN_COST);
15322   ins_encode %{
15323     __ subs(zr, $op1$$Register, $op2$$constant);
15324   %}
15325 
15326   ins_pipe(icmp_reg_imm);
15327 %}
15328 
15329 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15330 %{
15331   match(Set cr (OverflowSubI zero op1));
15332 
15333   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15334   ins_cost(INSN_COST);
15335   ins_encode %{
15336     __ cmpw(zr, $op1$$Register);
15337   %}
15338 
15339   ins_pipe(icmp_reg_imm);
15340 %}
15341 
15342 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15343 %{
15344   match(Set cr (OverflowSubL zero op1));
15345 
15346   format %{ "cmp   zr, $op1\t# overflow check long" %}
15347   ins_cost(INSN_COST);
15348   ins_encode %{
15349     __ cmp(zr, $op1$$Register);
15350   %}
15351 
15352   ins_pipe(icmp_reg_imm);
15353 %}
15354 
15355 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15356 %{
15357   match(Set cr (OverflowMulI op1 op2));
15358 
15359   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15360             "cmp   rscratch1, rscratch1, sxtw\n\t"
15361             "movw  rscratch1, #0x80000000\n\t"
15362             "cselw rscratch1, rscratch1, zr, NE\n\t"
15363             "cmpw  rscratch1, #1" %}
15364   ins_cost(5 * INSN_COST);
15365   ins_encode %{
15366     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15367     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15368     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15369     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15370     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15371   %}
15372 
15373   ins_pipe(pipe_slow);
15374 %}
15375 
15376 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15377 %{
15378   match(If cmp (OverflowMulI op1 op2));
15379   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15380             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15381   effect(USE labl, KILL cr);
15382 
15383   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15384             "cmp   rscratch1, rscratch1, sxtw\n\t"
15385             "b$cmp   $labl" %}
15386   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15387   ins_encode %{
15388     Label* L = $labl$$label;
15389     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15390     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15391     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15392     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15393   %}
15394 
15395   ins_pipe(pipe_serial);
15396 %}
15397 
15398 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15399 %{
15400   match(Set cr (OverflowMulL op1 op2));
15401 
15402   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15403             "smulh rscratch2, $op1, $op2\n\t"
15404             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15405             "movw  rscratch1, #0x80000000\n\t"
15406             "cselw rscratch1, rscratch1, zr, NE\n\t"
15407             "cmpw  rscratch1, #1" %}
15408   ins_cost(6 * INSN_COST);
15409   ins_encode %{
15410     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15411     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15412     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15413     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15414     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15415     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15416   %}
15417 
15418   ins_pipe(pipe_slow);
15419 %}
15420 
15421 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15422 %{
15423   match(If cmp (OverflowMulL op1 op2));
15424   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15425             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15426   effect(USE labl, KILL cr);
15427 
15428   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15429             "smulh rscratch2, $op1, $op2\n\t"
15430             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15431             "b$cmp $labl" %}
15432   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15433   ins_encode %{
15434     Label* L = $labl$$label;
15435     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15436     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15437     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15438     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15439     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15440   %}
15441 
15442   ins_pipe(pipe_serial);
15443 %}
15444 
15445 // ============================================================================
15446 // Compare Instructions
15447 
15448 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15449 %{
15450   match(Set cr (CmpI op1 op2));
15451 
15452   effect(DEF cr, USE op1, USE op2);
15453 
15454   ins_cost(INSN_COST);
15455   format %{ "cmpw  $op1, $op2" %}
15456 
15457   ins_encode(aarch64_enc_cmpw(op1, op2));
15458 
15459   ins_pipe(icmp_reg_reg);
15460 %}
15461 
15462 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15463 %{
15464   match(Set cr (CmpI op1 zero));
15465 
15466   effect(DEF cr, USE op1);
15467 
15468   ins_cost(INSN_COST);
15469   format %{ "cmpw $op1, 0" %}
15470 
15471   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15472 
15473   ins_pipe(icmp_reg_imm);
15474 %}
15475 
15476 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15477 %{
15478   match(Set cr (CmpI op1 op2));
15479 
15480   effect(DEF cr, USE op1);
15481 
15482   ins_cost(INSN_COST);
15483   format %{ "cmpw  $op1, $op2" %}
15484 
15485   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15486 
15487   ins_pipe(icmp_reg_imm);
15488 %}
15489 
15490 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15491 %{
15492   match(Set cr (CmpI op1 op2));
15493 
15494   effect(DEF cr, USE op1);
15495 
15496   ins_cost(INSN_COST * 2);
15497   format %{ "cmpw  $op1, $op2" %}
15498 
15499   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15500 
15501   ins_pipe(icmp_reg_imm);
15502 %}
15503 
15504 // Unsigned compare Instructions; really, same as signed compare
15505 // except it should only be used to feed an If or a CMovI which takes a
15506 // cmpOpU.
15507 
15508 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15509 %{
15510   match(Set cr (CmpU op1 op2));
15511 
15512   effect(DEF cr, USE op1, USE op2);
15513 
15514   ins_cost(INSN_COST);
15515   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15516 
15517   ins_encode(aarch64_enc_cmpw(op1, op2));
15518 
15519   ins_pipe(icmp_reg_reg);
15520 %}
15521 
15522 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15523 %{
15524   match(Set cr (CmpU op1 zero));
15525 
15526   effect(DEF cr, USE op1);
15527 
15528   ins_cost(INSN_COST);
15529   format %{ "cmpw $op1, #0\t# unsigned" %}
15530 
15531   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15532 
15533   ins_pipe(icmp_reg_imm);
15534 %}
15535 
15536 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15537 %{
15538   match(Set cr (CmpU op1 op2));
15539 
15540   effect(DEF cr, USE op1);
15541 
15542   ins_cost(INSN_COST);
15543   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15544 
15545   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15546 
15547   ins_pipe(icmp_reg_imm);
15548 %}
15549 
15550 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15551 %{
15552   match(Set cr (CmpU op1 op2));
15553 
15554   effect(DEF cr, USE op1);
15555 
15556   ins_cost(INSN_COST * 2);
15557   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15558 
15559   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15560 
15561   ins_pipe(icmp_reg_imm);
15562 %}
15563 
15564 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15565 %{
15566   match(Set cr (CmpL op1 op2));
15567 
15568   effect(DEF cr, USE op1, USE op2);
15569 
15570   ins_cost(INSN_COST);
15571   format %{ "cmp  $op1, $op2" %}
15572 
15573   ins_encode(aarch64_enc_cmp(op1, op2));
15574 
15575   ins_pipe(icmp_reg_reg);
15576 %}
15577 
15578 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15579 %{
15580   match(Set cr (CmpL op1 zero));
15581 
15582   effect(DEF cr, USE op1);
15583 
15584   ins_cost(INSN_COST);
15585   format %{ "tst  $op1" %}
15586 
15587   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15588 
15589   ins_pipe(icmp_reg_imm);
15590 %}
15591 
15592 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15593 %{
15594   match(Set cr (CmpL op1 op2));
15595 
15596   effect(DEF cr, USE op1);
15597 
15598   ins_cost(INSN_COST);
15599   format %{ "cmp  $op1, $op2" %}
15600 
15601   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15602 
15603   ins_pipe(icmp_reg_imm);
15604 %}
15605 
15606 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15607 %{
15608   match(Set cr (CmpL op1 op2));
15609 
15610   effect(DEF cr, USE op1);
15611 
15612   ins_cost(INSN_COST * 2);
15613   format %{ "cmp  $op1, $op2" %}
15614 
15615   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15616 
15617   ins_pipe(icmp_reg_imm);
15618 %}
15619 
15620 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15621 %{
15622   match(Set cr (CmpUL op1 op2));
15623 
15624   effect(DEF cr, USE op1, USE op2);
15625 
15626   ins_cost(INSN_COST);
15627   format %{ "cmp  $op1, $op2" %}
15628 
15629   ins_encode(aarch64_enc_cmp(op1, op2));
15630 
15631   ins_pipe(icmp_reg_reg);
15632 %}
15633 
15634 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15635 %{
15636   match(Set cr (CmpUL op1 zero));
15637 
15638   effect(DEF cr, USE op1);
15639 
15640   ins_cost(INSN_COST);
15641   format %{ "tst  $op1" %}
15642 
15643   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15644 
15645   ins_pipe(icmp_reg_imm);
15646 %}
15647 
15648 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15649 %{
15650   match(Set cr (CmpUL op1 op2));
15651 
15652   effect(DEF cr, USE op1);
15653 
15654   ins_cost(INSN_COST);
15655   format %{ "cmp  $op1, $op2" %}
15656 
15657   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15658 
15659   ins_pipe(icmp_reg_imm);
15660 %}
15661 
15662 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15663 %{
15664   match(Set cr (CmpUL op1 op2));
15665 
15666   effect(DEF cr, USE op1);
15667 
15668   ins_cost(INSN_COST * 2);
15669   format %{ "cmp  $op1, $op2" %}
15670 
15671   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15672 
15673   ins_pipe(icmp_reg_imm);
15674 %}
15675 
15676 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15677 %{
15678   match(Set cr (CmpP op1 op2));
15679 
15680   effect(DEF cr, USE op1, USE op2);
15681 
15682   ins_cost(INSN_COST);
15683   format %{ "cmp  $op1, $op2\t // ptr" %}
15684 
15685   ins_encode(aarch64_enc_cmpp(op1, op2));
15686 
15687   ins_pipe(icmp_reg_reg);
15688 %}
15689 
15690 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15691 %{
15692   match(Set cr (CmpN op1 op2));
15693 
15694   effect(DEF cr, USE op1, USE op2);
15695 
15696   ins_cost(INSN_COST);
15697   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15698 
15699   ins_encode(aarch64_enc_cmpn(op1, op2));
15700 
15701   ins_pipe(icmp_reg_reg);
15702 %}
15703 
15704 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15705 %{
15706   match(Set cr (CmpP op1 zero));
15707 
15708   effect(DEF cr, USE op1, USE zero);
15709 
15710   ins_cost(INSN_COST);
15711   format %{ "cmp  $op1, 0\t // ptr" %}
15712 
15713   ins_encode(aarch64_enc_testp(op1));
15714 
15715   ins_pipe(icmp_reg_imm);
15716 %}
15717 
15718 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15719 %{
15720   match(Set cr (CmpN op1 zero));
15721 
15722   effect(DEF cr, USE op1, USE zero);
15723 
15724   ins_cost(INSN_COST);
15725   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15726 
15727   ins_encode(aarch64_enc_testn(op1));
15728 
15729   ins_pipe(icmp_reg_imm);
15730 %}
15731 
15732 // FP comparisons
15733 //
15734 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15735 // using normal cmpOp. See declaration of rFlagsReg for details.
15736 
15737 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15738 %{
15739   match(Set cr (CmpF src1 src2));
15740 
15741   ins_cost(3 * INSN_COST);
15742   format %{ "fcmps $src1, $src2" %}
15743 
15744   ins_encode %{
15745     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15746   %}
15747 
15748   ins_pipe(pipe_class_compare);
15749 %}
15750 
15751 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15752 %{
15753   match(Set cr (CmpF src1 src2));
15754 
15755   ins_cost(3 * INSN_COST);
15756   format %{ "fcmps $src1, 0.0" %}
15757 
15758   ins_encode %{
15759     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15760   %}
15761 
15762   ins_pipe(pipe_class_compare);
15763 %}
15764 // FROM HERE
15765 
15766 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15767 %{
15768   match(Set cr (CmpD src1 src2));
15769 
15770   ins_cost(3 * INSN_COST);
15771   format %{ "fcmpd $src1, $src2" %}
15772 
15773   ins_encode %{
15774     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15775   %}
15776 
15777   ins_pipe(pipe_class_compare);
15778 %}
15779 
15780 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15781 %{
15782   match(Set cr (CmpD src1 src2));
15783 
15784   ins_cost(3 * INSN_COST);
15785   format %{ "fcmpd $src1, 0.0" %}
15786 
15787   ins_encode %{
15788     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15789   %}
15790 
15791   ins_pipe(pipe_class_compare);
15792 %}
15793 
15794 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15795 %{
15796   match(Set dst (CmpF3 src1 src2));
15797   effect(KILL cr);
15798 
15799   ins_cost(5 * INSN_COST);
15800   format %{ "fcmps $src1, $src2\n\t"
15801             "csinvw($dst, zr, zr, eq\n\t"
15802             "csnegw($dst, $dst, $dst, lt)"
15803   %}
15804 
15805   ins_encode %{
15806     Label done;
15807     FloatRegister s1 = as_FloatRegister($src1$$reg);
15808     FloatRegister s2 = as_FloatRegister($src2$$reg);
15809     Register d = as_Register($dst$$reg);
15810     __ fcmps(s1, s2);
15811     // installs 0 if EQ else -1
15812     __ csinvw(d, zr, zr, Assembler::EQ);
15813     // keeps -1 if less or unordered else installs 1
15814     __ csnegw(d, d, d, Assembler::LT);
15815     __ bind(done);
15816   %}
15817 
15818   ins_pipe(pipe_class_default);
15819 
15820 %}
15821 
15822 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15823 %{
15824   match(Set dst (CmpD3 src1 src2));
15825   effect(KILL cr);
15826 
15827   ins_cost(5 * INSN_COST);
15828   format %{ "fcmpd $src1, $src2\n\t"
15829             "csinvw($dst, zr, zr, eq\n\t"
15830             "csnegw($dst, $dst, $dst, lt)"
15831   %}
15832 
15833   ins_encode %{
15834     Label done;
15835     FloatRegister s1 = as_FloatRegister($src1$$reg);
15836     FloatRegister s2 = as_FloatRegister($src2$$reg);
15837     Register d = as_Register($dst$$reg);
15838     __ fcmpd(s1, s2);
15839     // installs 0 if EQ else -1
15840     __ csinvw(d, zr, zr, Assembler::EQ);
15841     // keeps -1 if less or unordered else installs 1
15842     __ csnegw(d, d, d, Assembler::LT);
15843     __ bind(done);
15844   %}
15845   ins_pipe(pipe_class_default);
15846 
15847 %}
15848 
15849 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15850 %{
15851   match(Set dst (CmpF3 src1 zero));
15852   effect(KILL cr);
15853 
15854   ins_cost(5 * INSN_COST);
15855   format %{ "fcmps $src1, 0.0\n\t"
15856             "csinvw($dst, zr, zr, eq\n\t"
15857             "csnegw($dst, $dst, $dst, lt)"
15858   %}
15859 
15860   ins_encode %{
15861     Label done;
15862     FloatRegister s1 = as_FloatRegister($src1$$reg);
15863     Register d = as_Register($dst$$reg);
15864     __ fcmps(s1, 0.0);
15865     // installs 0 if EQ else -1
15866     __ csinvw(d, zr, zr, Assembler::EQ);
15867     // keeps -1 if less or unordered else installs 1
15868     __ csnegw(d, d, d, Assembler::LT);
15869     __ bind(done);
15870   %}
15871 
15872   ins_pipe(pipe_class_default);
15873 
15874 %}
15875 
15876 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15877 %{
15878   match(Set dst (CmpD3 src1 zero));
15879   effect(KILL cr);
15880 
15881   ins_cost(5 * INSN_COST);
15882   format %{ "fcmpd $src1, 0.0\n\t"
15883             "csinvw($dst, zr, zr, eq\n\t"
15884             "csnegw($dst, $dst, $dst, lt)"
15885   %}
15886 
15887   ins_encode %{
15888     Label done;
15889     FloatRegister s1 = as_FloatRegister($src1$$reg);
15890     Register d = as_Register($dst$$reg);
15891     __ fcmpd(s1, 0.0);
15892     // installs 0 if EQ else -1
15893     __ csinvw(d, zr, zr, Assembler::EQ);
15894     // keeps -1 if less or unordered else installs 1
15895     __ csnegw(d, d, d, Assembler::LT);
15896     __ bind(done);
15897   %}
15898   ins_pipe(pipe_class_default);
15899 
15900 %}
15901 
15902 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15903 %{
15904   match(Set dst (CmpLTMask p q));
15905   effect(KILL cr);
15906 
15907   ins_cost(3 * INSN_COST);
15908 
15909   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15910             "csetw $dst, lt\n\t"
15911             "subw $dst, zr, $dst"
15912   %}
15913 
15914   ins_encode %{
15915     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15916     __ csetw(as_Register($dst$$reg), Assembler::LT);
15917     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15918   %}
15919 
15920   ins_pipe(ialu_reg_reg);
15921 %}
15922 
15923 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15924 %{
15925   match(Set dst (CmpLTMask src zero));
15926   effect(KILL cr);
15927 
15928   ins_cost(INSN_COST);
15929 
15930   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15931 
15932   ins_encode %{
15933     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15934   %}
15935 
15936   ins_pipe(ialu_reg_shift);
15937 %}
15938 
15939 // ============================================================================
15940 // Max and Min
15941 
15942 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15943 %{
15944   effect( DEF dst, USE src1, USE src2, USE cr );
15945 
15946   ins_cost(INSN_COST * 2);
15947   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15948 
15949   ins_encode %{
15950     __ cselw(as_Register($dst$$reg),
15951              as_Register($src1$$reg),
15952              as_Register($src2$$reg),
15953              Assembler::LT);
15954   %}
15955 
15956   ins_pipe(icond_reg_reg);
15957 %}
15958 
15959 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15960 %{
15961   match(Set dst (MinI src1 src2));
15962   ins_cost(INSN_COST * 3);
15963 
15964   expand %{
15965     rFlagsReg cr;
15966     compI_reg_reg(cr, src1, src2);
15967     cmovI_reg_reg_lt(dst, src1, src2, cr);
15968   %}
15969 
15970 %}
15971 // FROM HERE
15972 
15973 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15974 %{
15975   effect( DEF dst, USE src1, USE src2, USE cr );
15976 
15977   ins_cost(INSN_COST * 2);
15978   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15979 
15980   ins_encode %{
15981     __ cselw(as_Register($dst$$reg),
15982              as_Register($src1$$reg),
15983              as_Register($src2$$reg),
15984              Assembler::GT);
15985   %}
15986 
15987   ins_pipe(icond_reg_reg);
15988 %}
15989 
15990 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15991 %{
15992   match(Set dst (MaxI src1 src2));
15993   ins_cost(INSN_COST * 3);
15994   expand %{
15995     rFlagsReg cr;
15996     compI_reg_reg(cr, src1, src2);
15997     cmovI_reg_reg_gt(dst, src1, src2, cr);
15998   %}
15999 %}
16000 
16001 // ============================================================================
16002 // Branch Instructions
16003 
16004 // Direct Branch.
16005 instruct branch(label lbl)
16006 %{
16007   match(Goto);
16008 
16009   effect(USE lbl);
16010 
16011   ins_cost(BRANCH_COST);
16012   format %{ "b  $lbl" %}
16013 
16014   ins_encode(aarch64_enc_b(lbl));
16015 
16016   ins_pipe(pipe_branch);
16017 %}
16018 
16019 // Conditional Near Branch
16020 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16021 %{
16022   // Same match rule as `branchConFar'.
16023   match(If cmp cr);
16024 
16025   effect(USE lbl);
16026 
16027   ins_cost(BRANCH_COST);
16028   // If set to 1 this indicates that the current instruction is a
16029   // short variant of a long branch. This avoids using this
16030   // instruction in first-pass matching. It will then only be used in
16031   // the `Shorten_branches' pass.
16032   // ins_short_branch(1);
16033   format %{ "b$cmp  $lbl" %}
16034 
16035   ins_encode(aarch64_enc_br_con(cmp, lbl));
16036 
16037   ins_pipe(pipe_branch_cond);
16038 %}
16039 
16040 // Conditional Near Branch Unsigned
16041 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16042 %{
16043   // Same match rule as `branchConFar'.
16044   match(If cmp cr);
16045 
16046   effect(USE lbl);
16047 
16048   ins_cost(BRANCH_COST);
16049   // If set to 1 this indicates that the current instruction is a
16050   // short variant of a long branch. This avoids using this
16051   // instruction in first-pass matching. It will then only be used in
16052   // the `Shorten_branches' pass.
16053   // ins_short_branch(1);
16054   format %{ "b$cmp  $lbl\t# unsigned" %}
16055 
16056   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16057 
16058   ins_pipe(pipe_branch_cond);
16059 %}
16060 
16061 // Make use of CBZ and CBNZ.  These instructions, as well as being
16062 // shorter than (cmp; branch), have the additional benefit of not
16063 // killing the flags.
16064 
16065 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16066   match(If cmp (CmpI op1 op2));
16067   effect(USE labl);
16068 
16069   ins_cost(BRANCH_COST);
16070   format %{ "cbw$cmp   $op1, $labl" %}
16071   ins_encode %{
16072     Label* L = $labl$$label;
16073     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16074     if (cond == Assembler::EQ)
16075       __ cbzw($op1$$Register, *L);
16076     else
16077       __ cbnzw($op1$$Register, *L);
16078   %}
16079   ins_pipe(pipe_cmp_branch);
16080 %}
16081 
16082 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16083   match(If cmp (CmpL op1 op2));
16084   effect(USE labl);
16085 
16086   ins_cost(BRANCH_COST);
16087   format %{ "cb$cmp   $op1, $labl" %}
16088   ins_encode %{
16089     Label* L = $labl$$label;
16090     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16091     if (cond == Assembler::EQ)
16092       __ cbz($op1$$Register, *L);
16093     else
16094       __ cbnz($op1$$Register, *L);
16095   %}
16096   ins_pipe(pipe_cmp_branch);
16097 %}
16098 
16099 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16100   match(If cmp (CmpP op1 op2));
16101   effect(USE labl);
16102 
16103   ins_cost(BRANCH_COST);
16104   format %{ "cb$cmp   $op1, $labl" %}
16105   ins_encode %{
16106     Label* L = $labl$$label;
16107     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16108     if (cond == Assembler::EQ)
16109       __ cbz($op1$$Register, *L);
16110     else
16111       __ cbnz($op1$$Register, *L);
16112   %}
16113   ins_pipe(pipe_cmp_branch);
16114 %}
16115 
16116 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16117   match(If cmp (CmpN op1 op2));
16118   effect(USE labl);
16119 
16120   ins_cost(BRANCH_COST);
16121   format %{ "cbw$cmp   $op1, $labl" %}
16122   ins_encode %{
16123     Label* L = $labl$$label;
16124     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16125     if (cond == Assembler::EQ)
16126       __ cbzw($op1$$Register, *L);
16127     else
16128       __ cbnzw($op1$$Register, *L);
16129   %}
16130   ins_pipe(pipe_cmp_branch);
16131 %}
16132 
16133 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16134   match(If cmp (CmpP (DecodeN oop) zero));
16135   effect(USE labl);
16136 
16137   ins_cost(BRANCH_COST);
16138   format %{ "cb$cmp   $oop, $labl" %}
16139   ins_encode %{
16140     Label* L = $labl$$label;
16141     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16142     if (cond == Assembler::EQ)
16143       __ cbzw($oop$$Register, *L);
16144     else
16145       __ cbnzw($oop$$Register, *L);
16146   %}
16147   ins_pipe(pipe_cmp_branch);
16148 %}
16149 
16150 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16151   match(If cmp (CmpU op1 op2));
16152   effect(USE labl);
16153 
16154   ins_cost(BRANCH_COST);
16155   format %{ "cbw$cmp   $op1, $labl" %}
16156   ins_encode %{
16157     Label* L = $labl$$label;
16158     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16159     if (cond == Assembler::EQ || cond == Assembler::LS)
16160       __ cbzw($op1$$Register, *L);
16161     else
16162       __ cbnzw($op1$$Register, *L);
16163   %}
16164   ins_pipe(pipe_cmp_branch);
16165 %}
16166 
16167 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16168   match(If cmp (CmpUL op1 op2));
16169   effect(USE labl);
16170 
16171   ins_cost(BRANCH_COST);
16172   format %{ "cb$cmp   $op1, $labl" %}
16173   ins_encode %{
16174     Label* L = $labl$$label;
16175     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16176     if (cond == Assembler::EQ || cond == Assembler::LS)
16177       __ cbz($op1$$Register, *L);
16178     else
16179       __ cbnz($op1$$Register, *L);
16180   %}
16181   ins_pipe(pipe_cmp_branch);
16182 %}
16183 
16184 // Test bit and Branch
16185 
16186 // Patterns for short (< 32KiB) variants
16187 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16188   match(If cmp (CmpL op1 op2));
16189   effect(USE labl);
16190 
16191   ins_cost(BRANCH_COST);
16192   format %{ "cb$cmp   $op1, $labl # long" %}
16193   ins_encode %{
16194     Label* L = $labl$$label;
16195     Assembler::Condition cond =
16196       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16197     __ tbr(cond, $op1$$Register, 63, *L);
16198   %}
16199   ins_pipe(pipe_cmp_branch);
16200   ins_short_branch(1);
16201 %}
16202 
16203 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16204   match(If cmp (CmpI op1 op2));
16205   effect(USE labl);
16206 
16207   ins_cost(BRANCH_COST);
16208   format %{ "cb$cmp   $op1, $labl # int" %}
16209   ins_encode %{
16210     Label* L = $labl$$label;
16211     Assembler::Condition cond =
16212       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16213     __ tbr(cond, $op1$$Register, 31, *L);
16214   %}
16215   ins_pipe(pipe_cmp_branch);
16216   ins_short_branch(1);
16217 %}
16218 
16219 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16220   match(If cmp (CmpL (AndL op1 op2) op3));
16221   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16222   effect(USE labl);
16223 
16224   ins_cost(BRANCH_COST);
16225   format %{ "tb$cmp   $op1, $op2, $labl" %}
16226   ins_encode %{
16227     Label* L = $labl$$label;
16228     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16229     int bit = exact_log2_long($op2$$constant);
16230     __ tbr(cond, $op1$$Register, bit, *L);
16231   %}
16232   ins_pipe(pipe_cmp_branch);
16233   ins_short_branch(1);
16234 %}
16235 
16236 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16237   match(If cmp (CmpI (AndI op1 op2) op3));
16238   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16239   effect(USE labl);
16240 
16241   ins_cost(BRANCH_COST);
16242   format %{ "tb$cmp   $op1, $op2, $labl" %}
16243   ins_encode %{
16244     Label* L = $labl$$label;
16245     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16246     int bit = exact_log2((juint)$op2$$constant);
16247     __ tbr(cond, $op1$$Register, bit, *L);
16248   %}
16249   ins_pipe(pipe_cmp_branch);
16250   ins_short_branch(1);
16251 %}
16252 
16253 // And far variants
16254 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16255   match(If cmp (CmpL op1 op2));
16256   effect(USE labl);
16257 
16258   ins_cost(BRANCH_COST);
16259   format %{ "cb$cmp   $op1, $labl # long" %}
16260   ins_encode %{
16261     Label* L = $labl$$label;
16262     Assembler::Condition cond =
16263       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16264     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16265   %}
16266   ins_pipe(pipe_cmp_branch);
16267 %}
16268 
16269 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16270   match(If cmp (CmpI op1 op2));
16271   effect(USE labl);
16272 
16273   ins_cost(BRANCH_COST);
16274   format %{ "cb$cmp   $op1, $labl # int" %}
16275   ins_encode %{
16276     Label* L = $labl$$label;
16277     Assembler::Condition cond =
16278       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16279     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16280   %}
16281   ins_pipe(pipe_cmp_branch);
16282 %}
16283 
16284 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16285   match(If cmp (CmpL (AndL op1 op2) op3));
16286   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16287   effect(USE labl);
16288 
16289   ins_cost(BRANCH_COST);
16290   format %{ "tb$cmp   $op1, $op2, $labl" %}
16291   ins_encode %{
16292     Label* L = $labl$$label;
16293     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16294     int bit = exact_log2_long($op2$$constant);
16295     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16296   %}
16297   ins_pipe(pipe_cmp_branch);
16298 %}
16299 
16300 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16301   match(If cmp (CmpI (AndI op1 op2) op3));
16302   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16303   effect(USE labl);
16304 
16305   ins_cost(BRANCH_COST);
16306   format %{ "tb$cmp   $op1, $op2, $labl" %}
16307   ins_encode %{
16308     Label* L = $labl$$label;
16309     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16310     int bit = exact_log2((juint)$op2$$constant);
16311     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16312   %}
16313   ins_pipe(pipe_cmp_branch);
16314 %}
16315 
16316 // Test bits
16317 
16318 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16319   match(Set cr (CmpL (AndL op1 op2) op3));
16320   predicate(Assembler::operand_valid_for_logical_immediate
16321             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16322 
16323   ins_cost(INSN_COST);
16324   format %{ "tst $op1, $op2 # long" %}
16325   ins_encode %{
16326     __ tst($op1$$Register, $op2$$constant);
16327   %}
16328   ins_pipe(ialu_reg_reg);
16329 %}
16330 
16331 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16332   match(Set cr (CmpI (AndI op1 op2) op3));
16333   predicate(Assembler::operand_valid_for_logical_immediate
16334             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16335 
16336   ins_cost(INSN_COST);
16337   format %{ "tst $op1, $op2 # int" %}
16338   ins_encode %{
16339     __ tstw($op1$$Register, $op2$$constant);
16340   %}
16341   ins_pipe(ialu_reg_reg);
16342 %}
16343 
16344 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16345   match(Set cr (CmpL (AndL op1 op2) op3));
16346 
16347   ins_cost(INSN_COST);
16348   format %{ "tst $op1, $op2 # long" %}
16349   ins_encode %{
16350     __ tst($op1$$Register, $op2$$Register);
16351   %}
16352   ins_pipe(ialu_reg_reg);
16353 %}
16354 
16355 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16356   match(Set cr (CmpI (AndI op1 op2) op3));
16357 
16358   ins_cost(INSN_COST);
16359   format %{ "tstw $op1, $op2 # int" %}
16360   ins_encode %{
16361     __ tstw($op1$$Register, $op2$$Register);
16362   %}
16363   ins_pipe(ialu_reg_reg);
16364 %}
16365 
16366 
16367 // Conditional Far Branch
16368 // Conditional Far Branch Unsigned
16369 // TODO: fixme
16370 
16371 // counted loop end branch near
16372 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16373 %{
16374   match(CountedLoopEnd cmp cr);
16375 
16376   effect(USE lbl);
16377 
16378   ins_cost(BRANCH_COST);
16379   // short variant.
16380   // ins_short_branch(1);
16381   format %{ "b$cmp $lbl \t// counted loop end" %}
16382 
16383   ins_encode(aarch64_enc_br_con(cmp, lbl));
16384 
16385   ins_pipe(pipe_branch);
16386 %}
16387 
16388 // counted loop end branch near Unsigned
16389 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16390 %{
16391   match(CountedLoopEnd cmp cr);
16392 
16393   effect(USE lbl);
16394 
16395   ins_cost(BRANCH_COST);
16396   // short variant.
16397   // ins_short_branch(1);
16398   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16399 
16400   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16401 
16402   ins_pipe(pipe_branch);
16403 %}
16404 
16405 // counted loop end branch far
16406 // counted loop end branch far unsigned
16407 // TODO: fixme
16408 
16409 // ============================================================================
16410 // inlined locking and unlocking
16411 
16412 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16413 %{
16414   match(Set cr (FastLock object box));
16415   effect(TEMP tmp, TEMP tmp2);
16416 
16417   // TODO
16418   // identify correct cost
16419   ins_cost(5 * INSN_COST);
16420   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16421 
16422   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16423 
16424   ins_pipe(pipe_serial);
16425 %}
16426 
16427 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16428 %{
16429   match(Set cr (FastUnlock object box));
16430   effect(TEMP tmp, TEMP tmp2);
16431 
16432   ins_cost(5 * INSN_COST);
16433   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16434 
16435   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16436 
16437   ins_pipe(pipe_serial);
16438 %}
16439 
16440 
16441 // ============================================================================
16442 // Safepoint Instructions
16443 
16444 // TODO
16445 // provide a near and far version of this code
16446 
16447 instruct safePoint(rFlagsReg cr, iRegP poll)
16448 %{
16449   match(SafePoint poll);
16450   effect(KILL cr);
16451 
16452   format %{
16453     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16454   %}
16455   ins_encode %{
16456     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16457   %}
16458   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16459 %}
16460 
16461 
16462 // ============================================================================
16463 // Procedure Call/Return Instructions
16464 
16465 // Call Java Static Instruction
16466 
16467 instruct CallStaticJavaDirect(method meth)
16468 %{
16469   match(CallStaticJava);
16470 
16471   effect(USE meth);
16472 
16473   ins_cost(CALL_COST);
16474 
16475   format %{ "call,static $meth \t// ==> " %}
16476 
16477   ins_encode(aarch64_enc_java_static_call(meth),
16478              aarch64_enc_call_epilog);
16479 
16480   ins_pipe(pipe_class_call);
16481 %}
16482 
16483 // TO HERE
16484 
16485 // Call Java Dynamic Instruction
16486 instruct CallDynamicJavaDirect(method meth)
16487 %{
16488   match(CallDynamicJava);
16489 
16490   effect(USE meth);
16491 
16492   ins_cost(CALL_COST);
16493 
16494   format %{ "CALL,dynamic $meth \t// ==> " %}
16495 
16496   ins_encode(aarch64_enc_java_dynamic_call(meth),
16497              aarch64_enc_call_epilog);
16498 
16499   ins_pipe(pipe_class_call);
16500 %}
16501 
16502 // Call Runtime Instruction
16503 
16504 instruct CallRuntimeDirect(method meth)
16505 %{
16506   match(CallRuntime);
16507 
16508   effect(USE meth);
16509 
16510   ins_cost(CALL_COST);
16511 
16512   format %{ "CALL, runtime $meth" %}
16513 
16514   ins_encode( aarch64_enc_java_to_runtime(meth) );
16515 
16516   ins_pipe(pipe_class_call);
16517 %}
16518 
16519 // Call Runtime Instruction
16520 
16521 instruct CallLeafDirect(method meth)
16522 %{
16523   match(CallLeaf);
16524 
16525   effect(USE meth);
16526 
16527   ins_cost(CALL_COST);
16528 
16529   format %{ "CALL, runtime leaf $meth" %}
16530 
16531   ins_encode( aarch64_enc_java_to_runtime(meth) );
16532 
16533   ins_pipe(pipe_class_call);
16534 %}
16535 
16536 // Call Runtime Instruction
16537 
16538 instruct CallLeafNoFPDirect(method meth)
16539 %{
16540   match(CallLeafNoFP);
16541 
16542   effect(USE meth);
16543 
16544   ins_cost(CALL_COST);
16545 
16546   format %{ "CALL, runtime leaf nofp $meth" %}
16547 
16548   ins_encode( aarch64_enc_java_to_runtime(meth) );
16549 
16550   ins_pipe(pipe_class_call);
16551 %}
16552 
16553 instruct CallNativeDirect(method meth)
16554 %{
16555   match(CallNative);
16556 
16557   effect(USE meth);
16558 
16559   ins_cost(CALL_COST);
16560 
16561   format %{ "CALL, native $meth" %}
16562 
16563   ins_encode( aarch64_enc_java_to_runtime(meth) );
16564 
16565   ins_pipe(pipe_class_call);
16566 %}
16567 
16568 // Tail Call; Jump from runtime stub to Java code.
16569 // Also known as an 'interprocedural jump'.
16570 // Target of jump will eventually return to caller.
16571 // TailJump below removes the return address.
16572 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16573 %{
16574   match(TailCall jump_target method_ptr);
16575 
16576   ins_cost(CALL_COST);
16577 
16578   format %{ "br $jump_target\t# $method_ptr holds method" %}
16579 
16580   ins_encode(aarch64_enc_tail_call(jump_target));
16581 
16582   ins_pipe(pipe_class_call);
16583 %}
16584 
16585 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16586 %{
16587   match(TailJump jump_target ex_oop);
16588 
16589   ins_cost(CALL_COST);
16590 
16591   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16592 
16593   ins_encode(aarch64_enc_tail_jmp(jump_target));
16594 
16595   ins_pipe(pipe_class_call);
16596 %}
16597 
16598 // Create exception oop: created by stack-crawling runtime code.
16599 // Created exception is now available to this handler, and is setup
16600 // just prior to jumping to this handler. No code emitted.
16601 // TODO check
16602 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16603 instruct CreateException(iRegP_R0 ex_oop)
16604 %{
16605   match(Set ex_oop (CreateEx));
16606 
16607   format %{ " -- \t// exception oop; no code emitted" %}
16608 
16609   size(0);
16610 
16611   ins_encode( /*empty*/ );
16612 
16613   ins_pipe(pipe_class_empty);
16614 %}
16615 
16616 // Rethrow exception: The exception oop will come in the first
16617 // argument position. Then JUMP (not call) to the rethrow stub code.
16618 instruct RethrowException() %{
16619   match(Rethrow);
16620   ins_cost(CALL_COST);
16621 
16622   format %{ "b rethrow_stub" %}
16623 
16624   ins_encode( aarch64_enc_rethrow() );
16625 
16626   ins_pipe(pipe_class_call);
16627 %}
16628 
16629 
16630 // Return Instruction
16631 // epilog node loads ret address into lr as part of frame pop
16632 instruct Ret()
16633 %{
16634   match(Return);
16635 
16636   format %{ "ret\t// return register" %}
16637 
16638   ins_encode( aarch64_enc_ret() );
16639 
16640   ins_pipe(pipe_branch);
16641 %}
16642 
16643 // Die now.
16644 instruct ShouldNotReachHere() %{
16645   match(Halt);
16646 
16647   ins_cost(CALL_COST);
16648   format %{ "ShouldNotReachHere" %}
16649 
16650   ins_encode %{
16651     if (is_reachable()) {
16652       __ stop(_halt_reason);
16653     }
16654   %}
16655 
16656   ins_pipe(pipe_class_default);
16657 %}
16658 
16659 // ============================================================================
16660 // Partial Subtype Check
16661 //
16662 // superklass array for an instance of the superklass.  Set a hidden
16663 // internal cache on a hit (cache is checked with exposed code in
16664 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16665 // encoding ALSO sets flags.
16666 
16667 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16668 %{
16669   match(Set result (PartialSubtypeCheck sub super));
16670   effect(KILL cr, KILL temp);
16671 
16672   ins_cost(1100);  // slightly larger than the next version
16673   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16674 
16675   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16676 
16677   opcode(0x1); // Force zero of result reg on hit
16678 
16679   ins_pipe(pipe_class_memory);
16680 %}
16681 
16682 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16683 %{
16684   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16685   effect(KILL temp, KILL result);
16686 
16687   ins_cost(1100);  // slightly larger than the next version
16688   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16689 
16690   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16691 
16692   opcode(0x0); // Don't zero result reg on hit
16693 
16694   ins_pipe(pipe_class_memory);
16695 %}
16696 
16697 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16698                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16699 %{
16700   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16701   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16702   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16703 
16704   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16705   ins_encode %{
16706     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16707     __ string_compare($str1$$Register, $str2$$Register,
16708                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16709                       $tmp1$$Register, $tmp2$$Register,
16710                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16711   %}
16712   ins_pipe(pipe_class_memory);
16713 %}
16714 
16715 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16716                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16717 %{
16718   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16719   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16720   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16721 
16722   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16723   ins_encode %{
16724     __ string_compare($str1$$Register, $str2$$Register,
16725                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16726                       $tmp1$$Register, $tmp2$$Register,
16727                       fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16728   %}
16729   ins_pipe(pipe_class_memory);
16730 %}
16731 
16732 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16733                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16734                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16735 %{
16736   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16737   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16738   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16739          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16740 
16741   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16742   ins_encode %{
16743     __ string_compare($str1$$Register, $str2$$Register,
16744                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16745                       $tmp1$$Register, $tmp2$$Register,
16746                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16747                       $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16748   %}
16749   ins_pipe(pipe_class_memory);
16750 %}
16751 
16752 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16753                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16754                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16755 %{
16756   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16757   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16758   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16759          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16760 
16761   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16762   ins_encode %{
16763     __ string_compare($str1$$Register, $str2$$Register,
16764                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16765                       $tmp1$$Register, $tmp2$$Register,
16766                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16767                       $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16768   %}
16769   ins_pipe(pipe_class_memory);
16770 %}
16771 
16772 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16773        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16774        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16775 %{
16776   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16777   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16778   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16779          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16780   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16781 
16782   ins_encode %{
16783     __ string_indexof($str1$$Register, $str2$$Register,
16784                       $cnt1$$Register, $cnt2$$Register,
16785                       $tmp1$$Register, $tmp2$$Register,
16786                       $tmp3$$Register, $tmp4$$Register,
16787                       $tmp5$$Register, $tmp6$$Register,
16788                       -1, $result$$Register, StrIntrinsicNode::UU);
16789   %}
16790   ins_pipe(pipe_class_memory);
16791 %}
16792 
16793 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16794        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16795        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16796 %{
16797   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16798   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16799   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16800          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16801   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16802 
16803   ins_encode %{
16804     __ string_indexof($str1$$Register, $str2$$Register,
16805                       $cnt1$$Register, $cnt2$$Register,
16806                       $tmp1$$Register, $tmp2$$Register,
16807                       $tmp3$$Register, $tmp4$$Register,
16808                       $tmp5$$Register, $tmp6$$Register,
16809                       -1, $result$$Register, StrIntrinsicNode::LL);
16810   %}
16811   ins_pipe(pipe_class_memory);
16812 %}
16813 
16814 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16815        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16816        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16817 %{
16818   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16819   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16820   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16821          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16822   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16823 
16824   ins_encode %{
16825     __ string_indexof($str1$$Register, $str2$$Register,
16826                       $cnt1$$Register, $cnt2$$Register,
16827                       $tmp1$$Register, $tmp2$$Register,
16828                       $tmp3$$Register, $tmp4$$Register,
16829                       $tmp5$$Register, $tmp6$$Register,
16830                       -1, $result$$Register, StrIntrinsicNode::UL);
16831   %}
16832   ins_pipe(pipe_class_memory);
16833 %}
16834 
16835 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16836                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16837                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16838 %{
16839   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16840   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16841   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16842          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16843   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16844 
16845   ins_encode %{
16846     int icnt2 = (int)$int_cnt2$$constant;
16847     __ string_indexof($str1$$Register, $str2$$Register,
16848                       $cnt1$$Register, zr,
16849                       $tmp1$$Register, $tmp2$$Register,
16850                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16851                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16852   %}
16853   ins_pipe(pipe_class_memory);
16854 %}
16855 
16856 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16857                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16858                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16859 %{
16860   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16861   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16862   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16863          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16864   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16865 
16866   ins_encode %{
16867     int icnt2 = (int)$int_cnt2$$constant;
16868     __ string_indexof($str1$$Register, $str2$$Register,
16869                       $cnt1$$Register, zr,
16870                       $tmp1$$Register, $tmp2$$Register,
16871                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16872                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16873   %}
16874   ins_pipe(pipe_class_memory);
16875 %}
16876 
16877 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16878                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16879                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16880 %{
16881   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16882   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16883   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16884          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16885   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16886 
16887   ins_encode %{
16888     int icnt2 = (int)$int_cnt2$$constant;
16889     __ string_indexof($str1$$Register, $str2$$Register,
16890                       $cnt1$$Register, zr,
16891                       $tmp1$$Register, $tmp2$$Register,
16892                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16893                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16894   %}
16895   ins_pipe(pipe_class_memory);
16896 %}
16897 
16898 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16899                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16900                               iRegINoSp tmp3, rFlagsReg cr)
16901 %{
16902   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16903   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16904   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16905          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16906 
16907   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16908 
16909   ins_encode %{
16910     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16911                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16912                            $tmp3$$Register);
16913   %}
16914   ins_pipe(pipe_class_memory);
16915 %}
16916 
16917 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16918                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16919                               iRegINoSp tmp3, rFlagsReg cr)
16920 %{
16921   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16922   predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16923   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16924          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16925 
16926   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16927 
16928   ins_encode %{
16929     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16930                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16931                            $tmp3$$Register);
16932   %}
16933   ins_pipe(pipe_class_memory);
16934 %}
16935 
16936 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16937                         iRegI_R0 result, rFlagsReg cr)
16938 %{
16939   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16940   match(Set result (StrEquals (Binary str1 str2) cnt));
16941   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16942 
16943   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16944   ins_encode %{
16945     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16946     __ string_equals($str1$$Register, $str2$$Register,
16947                      $result$$Register, $cnt$$Register, 1);
16948   %}
16949   ins_pipe(pipe_class_memory);
16950 %}
16951 
16952 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16953                         iRegI_R0 result, rFlagsReg cr)
16954 %{
16955   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16956   match(Set result (StrEquals (Binary str1 str2) cnt));
16957   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16958 
16959   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16960   ins_encode %{
16961     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16962     __ string_equals($str1$$Register, $str2$$Register,
16963                      $result$$Register, $cnt$$Register, 2);
16964   %}
16965   ins_pipe(pipe_class_memory);
16966 %}
16967 
16968 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16969                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16970                        iRegP_R10 tmp, rFlagsReg cr)
16971 %{
16972   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16973   match(Set result (AryEq ary1 ary2));
16974   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16975 
16976   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16977   ins_encode %{
16978     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16979                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16980                                    $result$$Register, $tmp$$Register, 1);
16981     if (tpc == NULL) {
16982       ciEnv::current()->record_failure("CodeCache is full");
16983       return;
16984     }
16985   %}
16986   ins_pipe(pipe_class_memory);
16987 %}
16988 
16989 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16990                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16991                        iRegP_R10 tmp, rFlagsReg cr)
16992 %{
16993   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16994   match(Set result (AryEq ary1 ary2));
16995   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16996 
16997   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
16998   ins_encode %{
16999     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17000                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17001                                    $result$$Register, $tmp$$Register, 2);
17002     if (tpc == NULL) {
17003       ciEnv::current()->record_failure("CodeCache is full");
17004       return;
17005     }
17006   %}
17007   ins_pipe(pipe_class_memory);
17008 %}
17009 
17010 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17011 %{
17012   match(Set result (HasNegatives ary1 len));
17013   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17014   format %{ "has negatives byte[] $ary1,$len -> $result" %}
17015   ins_encode %{
17016     address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17017     if (tpc == NULL) {
17018       ciEnv::current()->record_failure("CodeCache is full");
17019       return;
17020     }
17021   %}
17022   ins_pipe( pipe_slow );
17023 %}
17024 
17025 // fast char[] to byte[] compression
17026 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17027                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17028                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17029                          iRegI_R0 result, rFlagsReg cr)
17030 %{
17031   match(Set result (StrCompressedCopy src (Binary dst len)));
17032   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17033 
17034   format %{ "String Compress $src,$dst -> $result    // KILL R1, R2, R3, R4" %}
17035   ins_encode %{
17036     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17037                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17038                            $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17039                            $result$$Register);
17040   %}
17041   ins_pipe( pipe_slow );
17042 %}
17043 
17044 // fast byte[] to char[] inflation
17045 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17046                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17047 %{
17048   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17049   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17050 
17051   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17052   ins_encode %{
17053     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17054                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17055                                         $tmp3$$FloatRegister, $tmp4$$Register);
17056     if (tpc == NULL) {
17057       ciEnv::current()->record_failure("CodeCache is full");
17058       return;
17059     }
17060   %}
17061   ins_pipe(pipe_class_memory);
17062 %}
17063 
17064 // encode char[] to byte[] in ISO_8859_1
17065 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17066                           vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17067                           vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17068                           iRegI_R0 result, rFlagsReg cr)
17069 %{
17070   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17071   match(Set result (EncodeISOArray src (Binary dst len)));
17072   effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17073          KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17074 
17075   format %{ "Encode array $src,$dst,$len -> $result" %}
17076   ins_encode %{
17077     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17078          $result$$Register, $Vtmp1$$FloatRegister,  $Vtmp2$$FloatRegister,
17079          $Vtmp3$$FloatRegister,  $Vtmp4$$FloatRegister);
17080   %}
17081   ins_pipe( pipe_class_memory );
17082 %}
17083 
17084 // ============================================================================
17085 // This name is KNOWN by the ADLC and cannot be changed.
17086 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17087 // for this guy.
17088 instruct tlsLoadP(thread_RegP dst)
17089 %{
17090   match(Set dst (ThreadLocal));
17091 
17092   ins_cost(0);
17093 
17094   format %{ " -- \t// $dst=Thread::current(), empty" %}
17095 
17096   size(0);
17097 
17098   ins_encode( /*empty*/ );
17099 
17100   ins_pipe(pipe_class_empty);
17101 %}
17102 
17103 //----------PEEPHOLE RULES-----------------------------------------------------
17104 // These must follow all instruction definitions as they use the names
17105 // defined in the instructions definitions.
17106 //
17107 // peepmatch ( root_instr_name [preceding_instruction]* );
17108 //
17109 // peepconstraint %{
17110 // (instruction_number.operand_name relational_op instruction_number.operand_name
17111 //  [, ...] );
17112 // // instruction numbers are zero-based using left to right order in peepmatch
17113 //
17114 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17115 // // provide an instruction_number.operand_name for each operand that appears
17116 // // in the replacement instruction's match rule
17117 //
17118 // ---------VM FLAGS---------------------------------------------------------
17119 //
17120 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17121 //
17122 // Each peephole rule is given an identifying number starting with zero and
17123 // increasing by one in the order seen by the parser.  An individual peephole
17124 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17125 // on the command-line.
17126 //
17127 // ---------CURRENT LIMITATIONS----------------------------------------------
17128 //
17129 // Only match adjacent instructions in same basic block
17130 // Only equality constraints
17131 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17132 // Only one replacement instruction
17133 //
17134 // ---------EXAMPLE----------------------------------------------------------
17135 //
17136 // // pertinent parts of existing instructions in architecture description
17137 // instruct movI(iRegINoSp dst, iRegI src)
17138 // %{
17139 //   match(Set dst (CopyI src));
17140 // %}
17141 //
17142 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17143 // %{
17144 //   match(Set dst (AddI dst src));
17145 //   effect(KILL cr);
17146 // %}
17147 //
17148 // // Change (inc mov) to lea
17149 // peephole %{
17150 //   // increment preceeded by register-register move
17151 //   peepmatch ( incI_iReg movI );
17152 //   // require that the destination register of the increment
17153 //   // match the destination register of the move
17154 //   peepconstraint ( 0.dst == 1.dst );
17155 //   // construct a replacement instruction that sets
17156 //   // the destination to ( move's source register + one )
17157 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17158 // %}
17159 //
17160 
17161 // Implementation no longer uses movX instructions since
17162 // machine-independent system no longer uses CopyX nodes.
17163 //
17164 // peephole
17165 // %{
17166 //   peepmatch (incI_iReg movI);
17167 //   peepconstraint (0.dst == 1.dst);
17168 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17169 // %}
17170 
17171 // peephole
17172 // %{
17173 //   peepmatch (decI_iReg movI);
17174 //   peepconstraint (0.dst == 1.dst);
17175 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17176 // %}
17177 
17178 // peephole
17179 // %{
17180 //   peepmatch (addI_iReg_imm movI);
17181 //   peepconstraint (0.dst == 1.dst);
17182 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17183 // %}
17184 
17185 // peephole
17186 // %{
17187 //   peepmatch (incL_iReg movL);
17188 //   peepconstraint (0.dst == 1.dst);
17189 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17190 // %}
17191 
17192 // peephole
17193 // %{
17194 //   peepmatch (decL_iReg movL);
17195 //   peepconstraint (0.dst == 1.dst);
17196 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17197 // %}
17198 
17199 // peephole
17200 // %{
17201 //   peepmatch (addL_iReg_imm movL);
17202 //   peepconstraint (0.dst == 1.dst);
17203 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17204 // %}
17205 
17206 // peephole
17207 // %{
17208 //   peepmatch (addP_iReg_imm movP);
17209 //   peepconstraint (0.dst == 1.dst);
17210 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17211 // %}
17212 
17213 // // Change load of spilled value to only a spill
17214 // instruct storeI(memory mem, iRegI src)
17215 // %{
17216 //   match(Set mem (StoreI mem src));
17217 // %}
17218 //
17219 // instruct loadI(iRegINoSp dst, memory mem)
17220 // %{
17221 //   match(Set dst (LoadI mem));
17222 // %}
17223 //
17224 
17225 //----------SMARTSPILL RULES---------------------------------------------------
17226 // These must follow all instruction definitions as they use the names
17227 // defined in the instructions definitions.
17228 
17229 // Local Variables:
17230 // mode: c++
17231 // End: