1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1134 
 1135 class CallStubImpl {
 1136 
 1137   //--------------------------------------------------------------
 1138   //---<  Used for optimization in Compile::shorten_branches  >---
 1139   //--------------------------------------------------------------
 1140 
 1141  public:
 1142   // Size of call trampoline stub.
 1143   static uint size_call_trampoline() {
 1144     return 0; // no call trampolines on this platform
 1145   }
 1146 
 1147   // number of relocations needed by a call trampoline stub
 1148   static uint reloc_call_trampoline() {
 1149     return 0; // no call trampolines on this platform
 1150   }
 1151 };
 1152 
 1153 class HandlerImpl {
 1154 
 1155  public:
 1156 
 1157   static int emit_exception_handler(C2_MacroAssembler *masm);
 1158   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1159 
 1160   static uint size_exception_handler() {
 1161     return MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 
 1164   static uint size_deopt_handler() {
 1165     // count one adr and one far branch instruction
 1166     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1167   }
 1168 };
 1169 
 1170 class Node::PD {
 1171 public:
 1172   enum NodeFlags {
 1173     _last_flag = Node::_last_flag
 1174   };
 1175 };
 1176 
 1177   bool is_CAS(int opcode, bool maybe_volatile);
 1178 
 1179   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1180 
 1181   bool unnecessary_acquire(const Node *barrier);
 1182   bool needs_acquiring_load(const Node *load);
 1183 
 1184   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1185 
 1186   bool unnecessary_release(const Node *barrier);
 1187   bool unnecessary_volatile(const Node *barrier);
 1188   bool needs_releasing_store(const Node *store);
 1189 
 1190   // predicate controlling translation of CompareAndSwapX
 1191   bool needs_acquiring_load_exclusive(const Node *load);
 1192 
 1193   // predicate controlling addressing modes
 1194   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1195 
 1196   // Convert BootTest condition to Assembler condition.
 1197   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1198   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1199 %}
 1200 
 1201 source %{
 1202 
 1203   // Derived RegMask with conditionally allocatable registers
 1204 
 1205   void PhaseOutput::pd_perform_mach_node_analysis() {
 1206   }
 1207 
 1208   int MachNode::pd_alignment_required() const {
 1209     return 1;
 1210   }
 1211 
 1212   int MachNode::compute_padding(int current_offset) const {
 1213     return 0;
 1214   }
 1215 
 1216   RegMask _ANY_REG32_mask;
 1217   RegMask _ANY_REG_mask;
 1218   RegMask _PTR_REG_mask;
 1219   RegMask _NO_SPECIAL_REG32_mask;
 1220   RegMask _NO_SPECIAL_REG_mask;
 1221   RegMask _NO_SPECIAL_PTR_REG_mask;
 1222   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1223 
 1224   void reg_mask_init() {
 1225     // We derive below RegMask(s) from the ones which are auto-generated from
 1226     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1227     // registers conditionally reserved.
 1228 
 1229     _ANY_REG32_mask = _ALL_REG32_mask;
 1230     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1231 
 1232     _ANY_REG_mask = _ALL_REG_mask;
 1233 
 1234     _PTR_REG_mask = _ALL_REG_mask;
 1235 
 1236     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1237     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1238 
 1239     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1240     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1241 
 1242     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1243     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1244 
 1245     // r27 is not allocatable when compressed oops is on and heapbase is not
 1246     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1247     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1251     }
 1252 
 1253     // r29 is not allocatable when PreserveFramePointer is on
 1254     if (PreserveFramePointer) {
 1255       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1256       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1257       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1258     }
 1259 
 1260     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1261     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1262   }
 1263 
 1264   // Optimizaton of volatile gets and puts
 1265   // -------------------------------------
 1266   //
 1267   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1268   // use to implement volatile reads and writes. For a volatile read
 1269   // we simply need
 1270   //
 1271   //   ldar<x>
 1272   //
 1273   // and for a volatile write we need
 1274   //
 1275   //   stlr<x>
 1276   //
 1277   // Alternatively, we can implement them by pairing a normal
 1278   // load/store with a memory barrier. For a volatile read we need
 1279   //
 1280   //   ldr<x>
 1281   //   dmb ishld
 1282   //
 1283   // for a volatile write
 1284   //
 1285   //   dmb ish
 1286   //   str<x>
 1287   //   dmb ish
 1288   //
 1289   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1290   // sequences. These are normally translated to an instruction
 1291   // sequence like the following
 1292   //
 1293   //   dmb      ish
 1294   // retry:
 1295   //   ldxr<x>   rval raddr
 1296   //   cmp       rval rold
 1297   //   b.ne done
 1298   //   stlxr<x>  rval, rnew, rold
 1299   //   cbnz      rval retry
 1300   // done:
 1301   //   cset      r0, eq
 1302   //   dmb ishld
 1303   //
 1304   // Note that the exclusive store is already using an stlxr
 1305   // instruction. That is required to ensure visibility to other
 1306   // threads of the exclusive write (assuming it succeeds) before that
 1307   // of any subsequent writes.
 1308   //
 1309   // The following instruction sequence is an improvement on the above
 1310   //
 1311   // retry:
 1312   //   ldaxr<x>  rval raddr
 1313   //   cmp       rval rold
 1314   //   b.ne done
 1315   //   stlxr<x>  rval, rnew, rold
 1316   //   cbnz      rval retry
 1317   // done:
 1318   //   cset      r0, eq
 1319   //
 1320   // We don't need the leading dmb ish since the stlxr guarantees
 1321   // visibility of prior writes in the case that the swap is
 1322   // successful. Crucially we don't have to worry about the case where
 1323   // the swap is not successful since no valid program should be
 1324   // relying on visibility of prior changes by the attempting thread
 1325   // in the case where the CAS fails.
 1326   //
 1327   // Similarly, we don't need the trailing dmb ishld if we substitute
 1328   // an ldaxr instruction since that will provide all the guarantees we
 1329   // require regarding observation of changes made by other threads
 1330   // before any change to the CAS address observed by the load.
 1331   //
 1332   // In order to generate the desired instruction sequence we need to
 1333   // be able to identify specific 'signature' ideal graph node
 1334   // sequences which i) occur as a translation of a volatile reads or
 1335   // writes or CAS operations and ii) do not occur through any other
 1336   // translation or graph transformation. We can then provide
 1337   // alternative aldc matching rules which translate these node
 1338   // sequences to the desired machine code sequences. Selection of the
 1339   // alternative rules can be implemented by predicates which identify
 1340   // the relevant node sequences.
 1341   //
 1342   // The ideal graph generator translates a volatile read to the node
 1343   // sequence
 1344   //
 1345   //   LoadX[mo_acquire]
 1346   //   MemBarAcquire
 1347   //
 1348   // As a special case when using the compressed oops optimization we
 1349   // may also see this variant
 1350   //
 1351   //   LoadN[mo_acquire]
 1352   //   DecodeN
 1353   //   MemBarAcquire
 1354   //
 1355   // A volatile write is translated to the node sequence
 1356   //
 1357   //   MemBarRelease
 1358   //   StoreX[mo_release] {CardMark}-optional
 1359   //   MemBarVolatile
 1360   //
 1361   // n.b. the above node patterns are generated with a strict
 1362   // 'signature' configuration of input and output dependencies (see
 1363   // the predicates below for exact details). The card mark may be as
 1364   // simple as a few extra nodes or, in a few GC configurations, may
 1365   // include more complex control flow between the leading and
 1366   // trailing memory barriers. However, whatever the card mark
 1367   // configuration these signatures are unique to translated volatile
 1368   // reads/stores -- they will not appear as a result of any other
 1369   // bytecode translation or inlining nor as a consequence of
 1370   // optimizing transforms.
 1371   //
 1372   // We also want to catch inlined unsafe volatile gets and puts and
 1373   // be able to implement them using either ldar<x>/stlr<x> or some
 1374   // combination of ldr<x>/stlr<x> and dmb instructions.
 1375   //
 1376   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1377   // normal volatile put node sequence containing an extra cpuorder
 1378   // membar
 1379   //
 1380   //   MemBarRelease
 1381   //   MemBarCPUOrder
 1382   //   StoreX[mo_release] {CardMark}-optional
 1383   //   MemBarCPUOrder
 1384   //   MemBarVolatile
 1385   //
 1386   // n.b. as an aside, a cpuorder membar is not itself subject to
 1387   // matching and translation by adlc rules.  However, the rule
 1388   // predicates need to detect its presence in order to correctly
 1389   // select the desired adlc rules.
 1390   //
 1391   // Inlined unsafe volatile gets manifest as a slightly different
 1392   // node sequence to a normal volatile get because of the
 1393   // introduction of some CPUOrder memory barriers to bracket the
 1394   // Load. However, but the same basic skeleton of a LoadX feeding a
 1395   // MemBarAcquire, possibly through an optional DecodeN, is still
 1396   // present
 1397   //
 1398   //   MemBarCPUOrder
 1399   //        ||       \\
 1400   //   MemBarCPUOrder LoadX[mo_acquire]
 1401   //        ||            |
 1402   //        ||       {DecodeN} optional
 1403   //        ||       /
 1404   //     MemBarAcquire
 1405   //
 1406   // In this case the acquire membar does not directly depend on the
 1407   // load. However, we can be sure that the load is generated from an
 1408   // inlined unsafe volatile get if we see it dependent on this unique
 1409   // sequence of membar nodes. Similarly, given an acquire membar we
 1410   // can know that it was added because of an inlined unsafe volatile
 1411   // get if it is fed and feeds a cpuorder membar and if its feed
 1412   // membar also feeds an acquiring load.
 1413   //
 1414   // Finally an inlined (Unsafe) CAS operation is translated to the
 1415   // following ideal graph
 1416   //
 1417   //   MemBarRelease
 1418   //   MemBarCPUOrder
 1419   //   CompareAndSwapX {CardMark}-optional
 1420   //   MemBarCPUOrder
 1421   //   MemBarAcquire
 1422   //
 1423   // So, where we can identify these volatile read and write
 1424   // signatures we can choose to plant either of the above two code
 1425   // sequences. For a volatile read we can simply plant a normal
 1426   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1427   // also choose to inhibit translation of the MemBarAcquire and
 1428   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1429   //
 1430   // When we recognise a volatile store signature we can choose to
 1431   // plant at a dmb ish as a translation for the MemBarRelease, a
 1432   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1433   // Alternatively, we can inhibit translation of the MemBarRelease
 1434   // and MemBarVolatile and instead plant a simple stlr<x>
 1435   // instruction.
 1436   //
 1437   // when we recognise a CAS signature we can choose to plant a dmb
 1438   // ish as a translation for the MemBarRelease, the conventional
 1439   // macro-instruction sequence for the CompareAndSwap node (which
 1440   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1441   // Alternatively, we can elide generation of the dmb instructions
 1442   // and plant the alternative CompareAndSwap macro-instruction
 1443   // sequence (which uses ldaxr<x>).
 1444   //
 1445   // Of course, the above only applies when we see these signature
 1446   // configurations. We still want to plant dmb instructions in any
 1447   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1448   // MemBarVolatile. For example, at the end of a constructor which
 1449   // writes final/volatile fields we will see a MemBarRelease
 1450   // instruction and this needs a 'dmb ish' lest we risk the
 1451   // constructed object being visible without making the
 1452   // final/volatile field writes visible.
 1453   //
 1454   // n.b. the translation rules below which rely on detection of the
 1455   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1456   // If we see anything other than the signature configurations we
 1457   // always just translate the loads and stores to ldr<x> and str<x>
 1458   // and translate acquire, release and volatile membars to the
 1459   // relevant dmb instructions.
 1460   //
 1461 
 1462   // is_CAS(int opcode, bool maybe_volatile)
 1463   //
 1464   // return true if opcode is one of the possible CompareAndSwapX
 1465   // values otherwise false.
 1466 
 1467   bool is_CAS(int opcode, bool maybe_volatile)
 1468   {
 1469     switch(opcode) {
 1470       // We handle these
 1471     case Op_CompareAndSwapI:
 1472     case Op_CompareAndSwapL:
 1473     case Op_CompareAndSwapP:
 1474     case Op_CompareAndSwapN:
 1475     case Op_ShenandoahCompareAndSwapP:
 1476     case Op_ShenandoahCompareAndSwapN:
 1477     case Op_CompareAndSwapB:
 1478     case Op_CompareAndSwapS:
 1479     case Op_GetAndSetI:
 1480     case Op_GetAndSetL:
 1481     case Op_GetAndSetP:
 1482     case Op_GetAndSetN:
 1483     case Op_GetAndAddI:
 1484     case Op_GetAndAddL:
 1485       return true;
 1486     case Op_CompareAndExchangeI:
 1487     case Op_CompareAndExchangeN:
 1488     case Op_CompareAndExchangeB:
 1489     case Op_CompareAndExchangeS:
 1490     case Op_CompareAndExchangeL:
 1491     case Op_CompareAndExchangeP:
 1492     case Op_WeakCompareAndSwapB:
 1493     case Op_WeakCompareAndSwapS:
 1494     case Op_WeakCompareAndSwapI:
 1495     case Op_WeakCompareAndSwapL:
 1496     case Op_WeakCompareAndSwapP:
 1497     case Op_WeakCompareAndSwapN:
 1498     case Op_ShenandoahWeakCompareAndSwapP:
 1499     case Op_ShenandoahWeakCompareAndSwapN:
 1500     case Op_ShenandoahCompareAndExchangeP:
 1501     case Op_ShenandoahCompareAndExchangeN:
 1502       return maybe_volatile;
 1503     default:
 1504       return false;
 1505     }
 1506   }
 1507 
 1508   // helper to determine the maximum number of Phi nodes we may need to
 1509   // traverse when searching from a card mark membar for the merge mem
 1510   // feeding a trailing membar or vice versa
 1511 
 1512 // predicates controlling emit of ldr<x>/ldar<x>
 1513 
 1514 bool unnecessary_acquire(const Node *barrier)
 1515 {
 1516   assert(barrier->is_MemBar(), "expecting a membar");
 1517 
 1518   MemBarNode* mb = barrier->as_MemBar();
 1519 
 1520   if (mb->trailing_load()) {
 1521     return true;
 1522   }
 1523 
 1524   if (mb->trailing_load_store()) {
 1525     Node* load_store = mb->in(MemBarNode::Precedent);
 1526     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1527     return is_CAS(load_store->Opcode(), true);
 1528   }
 1529 
 1530   return false;
 1531 }
 1532 
 1533 bool needs_acquiring_load(const Node *n)
 1534 {
 1535   assert(n->is_Load(), "expecting a load");
 1536   LoadNode *ld = n->as_Load();
 1537   return ld->is_acquire();
 1538 }
 1539 
 1540 bool unnecessary_release(const Node *n)
 1541 {
 1542   assert((n->is_MemBar() &&
 1543           n->Opcode() == Op_MemBarRelease),
 1544          "expecting a release membar");
 1545 
 1546   MemBarNode *barrier = n->as_MemBar();
 1547   if (!barrier->leading()) {
 1548     return false;
 1549   } else {
 1550     Node* trailing = barrier->trailing_membar();
 1551     MemBarNode* trailing_mb = trailing->as_MemBar();
 1552     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1553     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1554 
 1555     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1556     if (mem->is_Store()) {
 1557       assert(mem->as_Store()->is_release(), "");
 1558       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1559       return true;
 1560     } else {
 1561       assert(mem->is_LoadStore(), "");
 1562       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1563       return is_CAS(mem->Opcode(), true);
 1564     }
 1565   }
 1566   return false;
 1567 }
 1568 
 1569 bool unnecessary_volatile(const Node *n)
 1570 {
 1571   // assert n->is_MemBar();
 1572   MemBarNode *mbvol = n->as_MemBar();
 1573 
 1574   bool release = mbvol->trailing_store();
 1575   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1576 #ifdef ASSERT
 1577   if (release) {
 1578     Node* leading = mbvol->leading_membar();
 1579     assert(leading->Opcode() == Op_MemBarRelease, "");
 1580     assert(leading->as_MemBar()->leading_store(), "");
 1581     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1582   }
 1583 #endif
 1584 
 1585   return release;
 1586 }
 1587 
 1588 // predicates controlling emit of str<x>/stlr<x>
 1589 
 1590 bool needs_releasing_store(const Node *n)
 1591 {
 1592   // assert n->is_Store();
 1593   StoreNode *st = n->as_Store();
 1594   return st->trailing_membar() != nullptr;
 1595 }
 1596 
 1597 // predicate controlling translation of CAS
 1598 //
 1599 // returns true if CAS needs to use an acquiring load otherwise false
 1600 
 1601 bool needs_acquiring_load_exclusive(const Node *n)
 1602 {
 1603   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1604   LoadStoreNode* ldst = n->as_LoadStore();
 1605   if (is_CAS(n->Opcode(), false)) {
 1606     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1607   } else {
 1608     return ldst->trailing_membar() != nullptr;
 1609   }
 1610 
 1611   // so we can just return true here
 1612   return true;
 1613 }
 1614 
 1615 #define __ masm->
 1616 
 1617 // advance declarations for helper functions to convert register
 1618 // indices to register objects
 1619 
 1620 // the ad file has to provide implementations of certain methods
 1621 // expected by the generic code
 1622 //
 1623 // REQUIRED FUNCTIONALITY
 1624 
 1625 //=============================================================================
 1626 
 1627 // !!!!! Special hack to get all types of calls to specify the byte offset
 1628 //       from the start of the call to the point where the return address
 1629 //       will point.
 1630 
 1631 int MachCallStaticJavaNode::ret_addr_offset()
 1632 {
 1633   // call should be a simple bl
 1634   int off = 4;
 1635   return off;
 1636 }
 1637 
 1638 int MachCallDynamicJavaNode::ret_addr_offset()
 1639 {
 1640   return 16; // movz, movk, movk, bl
 1641 }
 1642 
 1643 int MachCallRuntimeNode::ret_addr_offset() {
 1644   // for generated stubs the call will be
 1645   //   bl(addr)
 1646   // or with far branches
 1647   //   bl(trampoline_stub)
 1648   // for real runtime callouts it will be six instructions
 1649   // see aarch64_enc_java_to_runtime
 1650   //   adr(rscratch2, retaddr)
 1651   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1652   //   lea(rscratch1, RuntimeAddress(addr)
 1653   //   blr(rscratch1)
 1654   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1655   if (cb) {
 1656     return 1 * NativeInstruction::instruction_size;
 1657   } else if (_entry_point == nullptr) {
 1658     // See CallLeafNoFPIndirect
 1659     return 1 * NativeInstruction::instruction_size;
 1660   } else {
 1661     return 6 * NativeInstruction::instruction_size;
 1662   }
 1663 }
 1664 
 1665 //=============================================================================
 1666 
 1667 #ifndef PRODUCT
 1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1669   st->print("BREAKPOINT");
 1670 }
 1671 #endif
 1672 
 1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1674   __ brk(0);
 1675 }
 1676 
 1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1678   return MachNode::size(ra_);
 1679 }
 1680 
 1681 //=============================================================================
 1682 
 1683 #ifndef PRODUCT
 1684   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1685     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1686   }
 1687 #endif
 1688 
 1689   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1690     for (int i = 0; i < _count; i++) {
 1691       __ nop();
 1692     }
 1693   }
 1694 
 1695   uint MachNopNode::size(PhaseRegAlloc*) const {
 1696     return _count * NativeInstruction::instruction_size;
 1697   }
 1698 
 1699 //=============================================================================
 1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1701 
 1702 int ConstantTable::calculate_table_base_offset() const {
 1703   return 0;  // absolute addressing, no offset
 1704 }
 1705 
 1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1708   ShouldNotReachHere();
 1709 }
 1710 
 1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1712   // Empty encoding
 1713 }
 1714 
 1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1716   return 0;
 1717 }
 1718 
 1719 #ifndef PRODUCT
 1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1721   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1722 }
 1723 #endif
 1724 
 1725 #ifndef PRODUCT
 1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1727   Compile* C = ra_->C;
 1728 
 1729   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1730 
 1731   if (C->output()->need_stack_bang(framesize))
 1732     st->print("# stack bang size=%d\n\t", framesize);
 1733 
 1734   if (VM_Version::use_rop_protection()) {
 1735     st->print("ldr  zr, [lr]\n\t");
 1736     st->print("paciaz\n\t");
 1737   }
 1738   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1739     st->print("sub  sp, sp, #%d\n\t", framesize);
 1740     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1741     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1742   } else {
 1743     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1744     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1745     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1746     st->print("sub  sp, sp, rscratch1");
 1747   }
 1748   if (C->stub_function() == nullptr) {
 1749     st->print("\n\t");
 1750     st->print("ldr  rscratch1, [guard]\n\t");
 1751     st->print("dmb ishld\n\t");
 1752     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1753     st->print("cmp  rscratch1, rscratch2\n\t");
 1754     st->print("b.eq skip");
 1755     st->print("\n\t");
 1756     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1757     st->print("b skip\n\t");
 1758     st->print("guard: int\n\t");
 1759     st->print("\n\t");
 1760     st->print("skip:\n\t");
 1761   }
 1762 }
 1763 #endif
 1764 
 1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1766   Compile* C = ra_->C;
 1767 
 1768   // insert a nop at the start of the prolog so we can patch in a
 1769   // branch if we need to invalidate the method later
 1770   __ nop();
 1771 
 1772   __ verified_entry(C, 0);
 1773 
 1774   if (C->stub_function() == nullptr) {
 1775     __ entry_barrier();
 1776   }
 1777 
 1778   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1779     __ bind(*_verified_entry);
 1780   }
 1781 
 1782   if (VerifyStackAtCalls) {
 1783     Unimplemented();
 1784   }
 1785 
 1786   C->output()->set_frame_complete(__ offset());
 1787 
 1788   if (C->has_mach_constant_base_node()) {
 1789     // NOTE: We set the table base offset here because users might be
 1790     // emitted before MachConstantBaseNode.
 1791     ConstantTable& constant_table = C->output()->constant_table();
 1792     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1793   }
 1794 }
 1795 
 1796 int MachPrologNode::reloc() const
 1797 {
 1798   return 0;
 1799 }
 1800 
 1801 //=============================================================================
 1802 
 1803 #ifndef PRODUCT
 1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1805   Compile* C = ra_->C;
 1806   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1807 
 1808   st->print("# pop frame %d\n\t",framesize);
 1809 
 1810   if (framesize == 0) {
 1811     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1812   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1813     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1814     st->print("add  sp, sp, #%d\n\t", framesize);
 1815   } else {
 1816     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1817     st->print("add  sp, sp, rscratch1\n\t");
 1818     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1819   }
 1820   if (VM_Version::use_rop_protection()) {
 1821     st->print("autiaz\n\t");
 1822     st->print("ldr  zr, [lr]\n\t");
 1823   }
 1824 
 1825   if (do_polling() && C->is_method_compilation()) {
 1826     st->print("# test polling word\n\t");
 1827     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1828     st->print("cmp  sp, rscratch1\n\t");
 1829     st->print("bhi #slow_path");
 1830   }
 1831 }
 1832 #endif
 1833 
 1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1835   Compile* C = ra_->C;
 1836   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1837 
 1838   __ remove_frame(framesize, C->needs_stack_repair());
 1839 
 1840   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1841     __ reserved_stack_check();
 1842   }
 1843 
 1844   if (do_polling() && C->is_method_compilation()) {
 1845     Label dummy_label;
 1846     Label* code_stub = &dummy_label;
 1847     if (!C->output()->in_scratch_emit_size()) {
 1848       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1849       C->output()->add_stub(stub);
 1850       code_stub = &stub->entry();
 1851     }
 1852     __ relocate(relocInfo::poll_return_type);
 1853     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1854   }
 1855 }
 1856 
 1857 int MachEpilogNode::reloc() const {
 1858   // Return number of relocatable values contained in this instruction.
 1859   return 1; // 1 for polling page.
 1860 }
 1861 
 1862 const Pipeline * MachEpilogNode::pipeline() const {
 1863   return MachNode::pipeline_class();
 1864 }
 1865 
 1866 //=============================================================================
 1867 
 1868 static enum RC rc_class(OptoReg::Name reg) {
 1869 
 1870   if (reg == OptoReg::Bad) {
 1871     return rc_bad;
 1872   }
 1873 
 1874   // we have 32 int registers * 2 halves
 1875   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1876 
 1877   if (reg < slots_of_int_registers) {
 1878     return rc_int;
 1879   }
 1880 
 1881   // we have 32 float register * 8 halves
 1882   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1883   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1884     return rc_float;
 1885   }
 1886 
 1887   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1888   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1889     return rc_predicate;
 1890   }
 1891 
 1892   // Between predicate regs & stack is the flags.
 1893   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1894 
 1895   return rc_stack;
 1896 }
 1897 
 1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1899   Compile* C = ra_->C;
 1900 
 1901   // Get registers to move.
 1902   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1903   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1904   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1905   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1906 
 1907   enum RC src_hi_rc = rc_class(src_hi);
 1908   enum RC src_lo_rc = rc_class(src_lo);
 1909   enum RC dst_hi_rc = rc_class(dst_hi);
 1910   enum RC dst_lo_rc = rc_class(dst_lo);
 1911 
 1912   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1913 
 1914   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1915     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1916            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1917            "expected aligned-adjacent pairs");
 1918   }
 1919 
 1920   if (src_lo == dst_lo && src_hi == dst_hi) {
 1921     return 0;            // Self copy, no move.
 1922   }
 1923 
 1924   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1925               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1926   int src_offset = ra_->reg2offset(src_lo);
 1927   int dst_offset = ra_->reg2offset(dst_lo);
 1928 
 1929   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1930     uint ireg = ideal_reg();
 1931     if (ireg == Op_VecA && masm) {
 1932       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1933       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1934         // stack->stack
 1935         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1936                                                 sve_vector_reg_size_in_bytes);
 1937       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1938         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1939                             sve_vector_reg_size_in_bytes);
 1940       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1941         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1942                               sve_vector_reg_size_in_bytes);
 1943       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1944         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1945                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1946                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1947       } else {
 1948         ShouldNotReachHere();
 1949       }
 1950     } else if (masm) {
 1951       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1952       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1953       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1954         // stack->stack
 1955         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1956         if (ireg == Op_VecD) {
 1957           __ unspill(rscratch1, true, src_offset);
 1958           __ spill(rscratch1, true, dst_offset);
 1959         } else {
 1960           __ spill_copy128(src_offset, dst_offset);
 1961         }
 1962       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1963         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1964                ireg == Op_VecD ? __ T8B : __ T16B,
 1965                as_FloatRegister(Matcher::_regEncode[src_lo]));
 1966       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1967         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 1968                  ireg == Op_VecD ? __ D : __ Q,
 1969                  ra_->reg2offset(dst_lo));
 1970       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1971         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1972                    ireg == Op_VecD ? __ D : __ Q,
 1973                    ra_->reg2offset(src_lo));
 1974       } else {
 1975         ShouldNotReachHere();
 1976       }
 1977     }
 1978   } else if (masm) {
 1979     switch (src_lo_rc) {
 1980     case rc_int:
 1981       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 1982         if (is64) {
 1983             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 1984                    as_Register(Matcher::_regEncode[src_lo]));
 1985         } else {
 1986             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 1987                     as_Register(Matcher::_regEncode[src_lo]));
 1988         }
 1989       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 1990         if (is64) {
 1991             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1992                      as_Register(Matcher::_regEncode[src_lo]));
 1993         } else {
 1994             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1995                      as_Register(Matcher::_regEncode[src_lo]));
 1996         }
 1997       } else {                    // gpr --> stack spill
 1998         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 1999         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2000       }
 2001       break;
 2002     case rc_float:
 2003       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2004         if (is64) {
 2005             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2006                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2007         } else {
 2008             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2009                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2010         }
 2011       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2012         if (is64) {
 2013             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2014                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2015         } else {
 2016             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2018         }
 2019       } else {                    // fpr --> stack spill
 2020         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2021         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2022                  is64 ? __ D : __ S, dst_offset);
 2023       }
 2024       break;
 2025     case rc_stack:
 2026       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2027         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2028       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2029         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2030                    is64 ? __ D : __ S, src_offset);
 2031       } else if (dst_lo_rc == rc_predicate) {
 2032         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2033                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2034       } else {                    // stack --> stack copy
 2035         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2036         if (ideal_reg() == Op_RegVectMask) {
 2037           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2038                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2039         } else {
 2040           __ unspill(rscratch1, is64, src_offset);
 2041           __ spill(rscratch1, is64, dst_offset);
 2042         }
 2043       }
 2044       break;
 2045     case rc_predicate:
 2046       if (dst_lo_rc == rc_predicate) {
 2047         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2048       } else if (dst_lo_rc == rc_stack) {
 2049         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2050                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2051       } else {
 2052         assert(false, "bad src and dst rc_class combination.");
 2053         ShouldNotReachHere();
 2054       }
 2055       break;
 2056     default:
 2057       assert(false, "bad rc_class for spill");
 2058       ShouldNotReachHere();
 2059     }
 2060   }
 2061 
 2062   if (st) {
 2063     st->print("spill ");
 2064     if (src_lo_rc == rc_stack) {
 2065       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2066     } else {
 2067       st->print("%s -> ", Matcher::regName[src_lo]);
 2068     }
 2069     if (dst_lo_rc == rc_stack) {
 2070       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2071     } else {
 2072       st->print("%s", Matcher::regName[dst_lo]);
 2073     }
 2074     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2075       int vsize = 0;
 2076       switch (ideal_reg()) {
 2077       case Op_VecD:
 2078         vsize = 64;
 2079         break;
 2080       case Op_VecX:
 2081         vsize = 128;
 2082         break;
 2083       case Op_VecA:
 2084         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2085         break;
 2086       default:
 2087         assert(false, "bad register type for spill");
 2088         ShouldNotReachHere();
 2089       }
 2090       st->print("\t# vector spill size = %d", vsize);
 2091     } else if (ideal_reg() == Op_RegVectMask) {
 2092       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2093       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2094       st->print("\t# predicate spill size = %d", vsize);
 2095     } else {
 2096       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2097     }
 2098   }
 2099 
 2100   return 0;
 2101 
 2102 }
 2103 
 2104 #ifndef PRODUCT
 2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2106   if (!ra_)
 2107     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2108   else
 2109     implementation(nullptr, ra_, false, st);
 2110 }
 2111 #endif
 2112 
 2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2114   implementation(masm, ra_, false, nullptr);
 2115 }
 2116 
 2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2118   return MachNode::size(ra_);
 2119 }
 2120 
 2121 //=============================================================================
 2122 
 2123 #ifndef PRODUCT
 2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2125   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2126   int reg = ra_->get_reg_first(this);
 2127   st->print("add %s, rsp, #%d]\t# box lock",
 2128             Matcher::regName[reg], offset);
 2129 }
 2130 #endif
 2131 
 2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2133   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2134   int reg    = ra_->get_encode(this);
 2135 
 2136   // This add will handle any 24-bit signed offset. 24 bits allows an
 2137   // 8 megabyte stack frame.
 2138   __ add(as_Register(reg), sp, offset);
 2139 }
 2140 
 2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2142   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2143   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2144 
 2145   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2146     return NativeInstruction::instruction_size;
 2147   } else {
 2148     return 2 * NativeInstruction::instruction_size;
 2149   }
 2150 }
 2151 
 2152 ///=============================================================================
 2153 #ifndef PRODUCT
 2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2155 {
 2156   st->print_cr("# MachVEPNode");
 2157   if (!_verified) {
 2158     st->print_cr("\t load_class");
 2159   } else {
 2160     st->print_cr("\t unpack_inline_arg");
 2161   }
 2162 }
 2163 #endif
 2164 
 2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
 2166 {
 2167   if (!_verified) {
 2168     __ ic_check(1);
 2169   } else {
 2170     // insert a nop at the start of the prolog so we can patch in a
 2171     // branch if we need to invalidate the method later
 2172     __ nop();
 2173 
 2174     // TODO 8284443 Avoid creation of temporary frame
 2175     if (ra_->C->stub_function() == nullptr) {
 2176       __ verified_entry(ra_->C, 0);
 2177       __ entry_barrier();
 2178       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2179       __ remove_frame(framesize, false);
 2180     }
 2181     // Unpack inline type args passed as oop and then jump to
 2182     // the verified entry point (skipping the unverified entry).
 2183     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2184     // Emit code for verified entry and save increment for stack repair on return
 2185     __ verified_entry(ra_->C, sp_inc);
 2186     if (Compile::current()->output()->in_scratch_emit_size()) {
 2187       Label dummy_verified_entry;
 2188       __ b(dummy_verified_entry);
 2189     } else {
 2190       __ b(*_verified_entry);
 2191     }
 2192   }
 2193 }
 2194 
 2195 //=============================================================================
 2196 #ifndef PRODUCT
 2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2198 {
 2199   st->print_cr("# MachUEPNode");
 2200   if (UseCompressedClassPointers) {
 2201     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2202     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2203     st->print_cr("\tcmpw rscratch1, r10");
 2204   } else {
 2205     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2206     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2207     st->print_cr("\tcmp rscratch1, r10");
 2208   }
 2209   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2210 }
 2211 #endif
 2212 
 2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2214 {
 2215   __ ic_check(InteriorEntryAlignment);
 2216 }
 2217 
 2218 // REQUIRED EMIT CODE
 2219 
 2220 //=============================================================================
 2221 
 2222 // Emit exception handler code.
 2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2224 {
 2225   // mov rscratch1 #exception_blob_entry_point
 2226   // br rscratch1
 2227   // Note that the code buffer's insts_mark is always relative to insts.
 2228   // That's why we must use the macroassembler to generate a handler.
 2229   address base = __ start_a_stub(size_exception_handler());
 2230   if (base == nullptr) {
 2231     ciEnv::current()->record_failure("CodeCache is full");
 2232     return 0;  // CodeBuffer::expand failed
 2233   }
 2234   int offset = __ offset();
 2235   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2236   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2237   __ end_a_stub();
 2238   return offset;
 2239 }
 2240 
 2241 // Emit deopt handler code.
 2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2243 {
 2244   // Note that the code buffer's insts_mark is always relative to insts.
 2245   // That's why we must use the macroassembler to generate a handler.
 2246   address base = __ start_a_stub(size_deopt_handler());
 2247   if (base == nullptr) {
 2248     ciEnv::current()->record_failure("CodeCache is full");
 2249     return 0;  // CodeBuffer::expand failed
 2250   }
 2251   int offset = __ offset();
 2252 
 2253   __ adr(lr, __ pc());
 2254   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2255 
 2256   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2257   __ end_a_stub();
 2258   return offset;
 2259 }
 2260 
 2261 // REQUIRED MATCHER CODE
 2262 
 2263 //=============================================================================
 2264 
 2265 bool Matcher::match_rule_supported(int opcode) {
 2266   if (!has_match_rule(opcode))
 2267     return false;
 2268 
 2269   switch (opcode) {
 2270     case Op_OnSpinWait:
 2271       return VM_Version::supports_on_spin_wait();
 2272     case Op_CacheWB:
 2273     case Op_CacheWBPreSync:
 2274     case Op_CacheWBPostSync:
 2275       if (!VM_Version::supports_data_cache_line_flush()) {
 2276         return false;
 2277       }
 2278       break;
 2279     case Op_ExpandBits:
 2280     case Op_CompressBits:
 2281       if (!VM_Version::supports_svebitperm()) {
 2282         return false;
 2283       }
 2284       break;
 2285     case Op_FmaF:
 2286     case Op_FmaD:
 2287     case Op_FmaVF:
 2288     case Op_FmaVD:
 2289       if (!UseFMA) {
 2290         return false;
 2291       }
 2292       break;
 2293   }
 2294 
 2295   return true; // Per default match rules are supported.
 2296 }
 2297 
 2298 const RegMask* Matcher::predicate_reg_mask(void) {
 2299   return &_PR_REG_mask;
 2300 }
 2301 
 2302 bool Matcher::supports_vector_calling_convention(void) {
 2303   return EnableVectorSupport && UseVectorStubs;
 2304 }
 2305 
 2306 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2307   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 2308   int lo = V0_num;
 2309   int hi = V0_H_num;
 2310   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2311     hi = V0_K_num;
 2312   }
 2313   return OptoRegPair(hi, lo);
 2314 }
 2315 
 2316 // Is this branch offset short enough that a short branch can be used?
 2317 //
 2318 // NOTE: If the platform does not provide any short branch variants, then
 2319 //       this method should return false for offset 0.
 2320 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2321   // The passed offset is relative to address of the branch.
 2322 
 2323   return (-32768 <= offset && offset < 32768);
 2324 }
 2325 
 2326 // Vector width in bytes.
 2327 int Matcher::vector_width_in_bytes(BasicType bt) {
 2328   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2329   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2330   // Minimum 2 values in vector
 2331   if (size < 2*type2aelembytes(bt)) size = 0;
 2332   // But never < 4
 2333   if (size < 4) size = 0;
 2334   return size;
 2335 }
 2336 
 2337 // Limits on vector size (number of elements) loaded into vector.
 2338 int Matcher::max_vector_size(const BasicType bt) {
 2339   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2340 }
 2341 
 2342 int Matcher::min_vector_size(const BasicType bt) {
 2343   int max_size = max_vector_size(bt);
 2344   // Limit the min vector size to 8 bytes.
 2345   int size = 8 / type2aelembytes(bt);
 2346   if (bt == T_BYTE) {
 2347     // To support vector api shuffle/rearrange.
 2348     size = 4;
 2349   } else if (bt == T_BOOLEAN) {
 2350     // To support vector api load/store mask.
 2351     size = 2;
 2352   }
 2353   if (size < 2) size = 2;
 2354   return MIN2(size, max_size);
 2355 }
 2356 
 2357 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2358   return Matcher::max_vector_size(bt);
 2359 }
 2360 
 2361 // Actual max scalable vector register length.
 2362 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2363   return Matcher::max_vector_size(bt);
 2364 }
 2365 
 2366 // Vector ideal reg.
 2367 uint Matcher::vector_ideal_reg(int len) {
 2368   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2369     return Op_VecA;
 2370   }
 2371   switch(len) {
 2372     // For 16-bit/32-bit mask vector, reuse VecD.
 2373     case  2:
 2374     case  4:
 2375     case  8: return Op_VecD;
 2376     case 16: return Op_VecX;
 2377   }
 2378   ShouldNotReachHere();
 2379   return 0;
 2380 }
 2381 
 2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2383   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2384   switch (ideal_reg) {
 2385     case Op_VecA: return new vecAOper();
 2386     case Op_VecD: return new vecDOper();
 2387     case Op_VecX: return new vecXOper();
 2388   }
 2389   ShouldNotReachHere();
 2390   return nullptr;
 2391 }
 2392 
 2393 bool Matcher::is_reg2reg_move(MachNode* m) {
 2394   return false;
 2395 }
 2396 
 2397 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2398   return opnd->opcode() == VREG;
 2399 }
 2400 
 2401 // Return whether or not this register is ever used as an argument.
 2402 // This function is used on startup to build the trampoline stubs in
 2403 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2404 // call in the trampoline, and arguments in those registers not be
 2405 // available to the callee.
 2406 bool Matcher::can_be_java_arg(int reg)
 2407 {
 2408   return
 2409     reg ==  R0_num || reg == R0_H_num ||
 2410     reg ==  R1_num || reg == R1_H_num ||
 2411     reg ==  R2_num || reg == R2_H_num ||
 2412     reg ==  R3_num || reg == R3_H_num ||
 2413     reg ==  R4_num || reg == R4_H_num ||
 2414     reg ==  R5_num || reg == R5_H_num ||
 2415     reg ==  R6_num || reg == R6_H_num ||
 2416     reg ==  R7_num || reg == R7_H_num ||
 2417     reg ==  V0_num || reg == V0_H_num ||
 2418     reg ==  V1_num || reg == V1_H_num ||
 2419     reg ==  V2_num || reg == V2_H_num ||
 2420     reg ==  V3_num || reg == V3_H_num ||
 2421     reg ==  V4_num || reg == V4_H_num ||
 2422     reg ==  V5_num || reg == V5_H_num ||
 2423     reg ==  V6_num || reg == V6_H_num ||
 2424     reg ==  V7_num || reg == V7_H_num;
 2425 }
 2426 
 2427 bool Matcher::is_spillable_arg(int reg)
 2428 {
 2429   return can_be_java_arg(reg);
 2430 }
 2431 
 2432 uint Matcher::int_pressure_limit()
 2433 {
 2434   // JDK-8183543: When taking the number of available registers as int
 2435   // register pressure threshold, the jtreg test:
 2436   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2437   // failed due to C2 compilation failure with
 2438   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2439   //
 2440   // A derived pointer is live at CallNode and then is flagged by RA
 2441   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2442   // derived pointers and lastly fail to spill after reaching maximum
 2443   // number of iterations. Lowering the default pressure threshold to
 2444   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2445   // a high register pressure area of the code so that split_DEF can
 2446   // generate DefinitionSpillCopy for the derived pointer.
 2447   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2448   if (!PreserveFramePointer) {
 2449     // When PreserveFramePointer is off, frame pointer is allocatable,
 2450     // but different from other SOC registers, it is excluded from
 2451     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2452     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2453     // See check_pressure_at_fatproj().
 2454     default_int_pressure_threshold--;
 2455   }
 2456   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2457 }
 2458 
 2459 uint Matcher::float_pressure_limit()
 2460 {
 2461   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2462   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2463 }
 2464 
 2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2466   return false;
 2467 }
 2468 
 2469 RegMask Matcher::divI_proj_mask() {
 2470   ShouldNotReachHere();
 2471   return RegMask();
 2472 }
 2473 
 2474 // Register for MODI projection of divmodI.
 2475 RegMask Matcher::modI_proj_mask() {
 2476   ShouldNotReachHere();
 2477   return RegMask();
 2478 }
 2479 
 2480 // Register for DIVL projection of divmodL.
 2481 RegMask Matcher::divL_proj_mask() {
 2482   ShouldNotReachHere();
 2483   return RegMask();
 2484 }
 2485 
 2486 // Register for MODL projection of divmodL.
 2487 RegMask Matcher::modL_proj_mask() {
 2488   ShouldNotReachHere();
 2489   return RegMask();
 2490 }
 2491 
 2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2493   return FP_REG_mask();
 2494 }
 2495 
 2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2497   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2498     Node* u = addp->fast_out(i);
 2499     if (u->is_LoadStore()) {
 2500       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2501       // instructions) only take register indirect as an operand, so
 2502       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2503       // must fail.
 2504       return false;
 2505     }
 2506     if (u->is_Mem()) {
 2507       int opsize = u->as_Mem()->memory_size();
 2508       assert(opsize > 0, "unexpected memory operand size");
 2509       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2510         return false;
 2511       }
 2512     }
 2513   }
 2514   return true;
 2515 }
 2516 
 2517 // Convert BootTest condition to Assembler condition.
 2518 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2519 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2520   Assembler::Condition result;
 2521   switch(cond) {
 2522     case BoolTest::eq:
 2523       result = Assembler::EQ; break;
 2524     case BoolTest::ne:
 2525       result = Assembler::NE; break;
 2526     case BoolTest::le:
 2527       result = Assembler::LE; break;
 2528     case BoolTest::ge:
 2529       result = Assembler::GE; break;
 2530     case BoolTest::lt:
 2531       result = Assembler::LT; break;
 2532     case BoolTest::gt:
 2533       result = Assembler::GT; break;
 2534     case BoolTest::ule:
 2535       result = Assembler::LS; break;
 2536     case BoolTest::uge:
 2537       result = Assembler::HS; break;
 2538     case BoolTest::ult:
 2539       result = Assembler::LO; break;
 2540     case BoolTest::ugt:
 2541       result = Assembler::HI; break;
 2542     case BoolTest::overflow:
 2543       result = Assembler::VS; break;
 2544     case BoolTest::no_overflow:
 2545       result = Assembler::VC; break;
 2546     default:
 2547       ShouldNotReachHere();
 2548       return Assembler::Condition(-1);
 2549   }
 2550 
 2551   // Check conversion
 2552   if (cond & BoolTest::unsigned_compare) {
 2553     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2554   } else {
 2555     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2556   }
 2557 
 2558   return result;
 2559 }
 2560 
 2561 // Binary src (Replicate con)
 2562 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2563   if (n == nullptr || m == nullptr) {
 2564     return false;
 2565   }
 2566 
 2567   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2568     return false;
 2569   }
 2570 
 2571   Node* imm_node = m->in(1);
 2572   if (!imm_node->is_Con()) {
 2573     return false;
 2574   }
 2575 
 2576   const Type* t = imm_node->bottom_type();
 2577   if (!(t->isa_int() || t->isa_long())) {
 2578     return false;
 2579   }
 2580 
 2581   switch (n->Opcode()) {
 2582   case Op_AndV:
 2583   case Op_OrV:
 2584   case Op_XorV: {
 2585     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2586     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2587     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2588   }
 2589   case Op_AddVB:
 2590     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2591   case Op_AddVS:
 2592   case Op_AddVI:
 2593     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2594   case Op_AddVL:
 2595     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2596   default:
 2597     return false;
 2598   }
 2599 }
 2600 
 2601 // (XorV src (Replicate m1))
 2602 // (XorVMask src (MaskAll m1))
 2603 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2604   if (n != nullptr && m != nullptr) {
 2605     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2606            VectorNode::is_all_ones_vector(m);
 2607   }
 2608   return false;
 2609 }
 2610 
 2611 // Should the matcher clone input 'm' of node 'n'?
 2612 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2613   if (is_vshift_con_pattern(n, m) ||
 2614       is_vector_bitwise_not_pattern(n, m) ||
 2615       is_valid_sve_arith_imm_pattern(n, m) ||
 2616       is_encode_and_store_pattern(n, m)) {
 2617     mstack.push(m, Visit);
 2618     return true;
 2619   }
 2620   return false;
 2621 }
 2622 
 2623 // Should the Matcher clone shifts on addressing modes, expecting them
 2624 // to be subsumed into complex addressing expressions or compute them
 2625 // into registers?
 2626 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2627 
 2628   // Loads and stores with indirect memory input (e.g., volatile loads and
 2629   // stores) do not subsume the input into complex addressing expressions. If
 2630   // the addressing expression is input to at least one such load or store, do
 2631   // not clone the addressing expression. Query needs_acquiring_load and
 2632   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2633   // possible to directly query for indirect memory input at this stage.
 2634   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2635     Node* n = m->fast_out(i);
 2636     if (n->is_Load() && needs_acquiring_load(n)) {
 2637       return false;
 2638     }
 2639     if (n->is_Store() && needs_releasing_store(n)) {
 2640       return false;
 2641     }
 2642   }
 2643 
 2644   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2645     return true;
 2646   }
 2647 
 2648   Node *off = m->in(AddPNode::Offset);
 2649   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2650       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2651       // Are there other uses besides address expressions?
 2652       !is_visited(off)) {
 2653     address_visited.set(off->_idx); // Flag as address_visited
 2654     mstack.push(off->in(2), Visit);
 2655     Node *conv = off->in(1);
 2656     if (conv->Opcode() == Op_ConvI2L &&
 2657         // Are there other uses besides address expressions?
 2658         !is_visited(conv)) {
 2659       address_visited.set(conv->_idx); // Flag as address_visited
 2660       mstack.push(conv->in(1), Pre_Visit);
 2661     } else {
 2662       mstack.push(conv, Pre_Visit);
 2663     }
 2664     address_visited.test_set(m->_idx); // Flag as address_visited
 2665     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2666     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2667     return true;
 2668   } else if (off->Opcode() == Op_ConvI2L &&
 2669              // Are there other uses besides address expressions?
 2670              !is_visited(off)) {
 2671     address_visited.test_set(m->_idx); // Flag as address_visited
 2672     address_visited.set(off->_idx); // Flag as address_visited
 2673     mstack.push(off->in(1), Pre_Visit);
 2674     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2675     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2676     return true;
 2677   }
 2678   return false;
 2679 }
 2680 
 2681 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2682   {                                                                     \
 2683     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2684     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2685     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2686     __ INSN(REG, as_Register(BASE));                                    \
 2687   }
 2688 
 2689 
 2690 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2691   {
 2692     Address::extend scale;
 2693 
 2694     // Hooboy, this is fugly.  We need a way to communicate to the
 2695     // encoder that the index needs to be sign extended, so we have to
 2696     // enumerate all the cases.
 2697     switch (opcode) {
 2698     case INDINDEXSCALEDI2L:
 2699     case INDINDEXSCALEDI2LN:
 2700     case INDINDEXI2L:
 2701     case INDINDEXI2LN:
 2702       scale = Address::sxtw(size);
 2703       break;
 2704     default:
 2705       scale = Address::lsl(size);
 2706     }
 2707 
 2708     if (index == -1) {
 2709       return Address(base, disp);
 2710     } else {
 2711       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2712       return Address(base, as_Register(index), scale);
 2713     }
 2714   }
 2715 
 2716 
 2717 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2718 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2719 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2720 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2721                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2722 
 2723   // Used for all non-volatile memory accesses.  The use of
 2724   // $mem->opcode() to discover whether this pattern uses sign-extended
 2725   // offsets is something of a kludge.
 2726   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2727                         Register reg, int opcode,
 2728                         Register base, int index, int scale, int disp,
 2729                         int size_in_memory)
 2730   {
 2731     Address addr = mem2address(opcode, base, index, scale, disp);
 2732     if (addr.getMode() == Address::base_plus_offset) {
 2733       /* Fix up any out-of-range offsets. */
 2734       assert_different_registers(rscratch1, base);
 2735       assert_different_registers(rscratch1, reg);
 2736       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2737     }
 2738     (masm->*insn)(reg, addr);
 2739   }
 2740 
 2741   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2742                         FloatRegister reg, int opcode,
 2743                         Register base, int index, int size, int disp,
 2744                         int size_in_memory)
 2745   {
 2746     Address::extend scale;
 2747 
 2748     switch (opcode) {
 2749     case INDINDEXSCALEDI2L:
 2750     case INDINDEXSCALEDI2LN:
 2751       scale = Address::sxtw(size);
 2752       break;
 2753     default:
 2754       scale = Address::lsl(size);
 2755     }
 2756 
 2757     if (index == -1) {
 2758       // Fix up any out-of-range offsets.
 2759       assert_different_registers(rscratch1, base);
 2760       Address addr = Address(base, disp);
 2761       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2762       (masm->*insn)(reg, addr);
 2763     } else {
 2764       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2765       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2766     }
 2767   }
 2768 
 2769   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2770                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2771                         int opcode, Register base, int index, int size, int disp)
 2772   {
 2773     if (index == -1) {
 2774       (masm->*insn)(reg, T, Address(base, disp));
 2775     } else {
 2776       assert(disp == 0, "unsupported address mode");
 2777       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2778     }
 2779   }
 2780 
 2781 %}
 2782 
 2783 
 2784 
 2785 //----------ENCODING BLOCK-----------------------------------------------------
 2786 // This block specifies the encoding classes used by the compiler to
 2787 // output byte streams.  Encoding classes are parameterized macros
 2788 // used by Machine Instruction Nodes in order to generate the bit
 2789 // encoding of the instruction.  Operands specify their base encoding
 2790 // interface with the interface keyword.  There are currently
 2791 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2792 // COND_INTER.  REG_INTER causes an operand to generate a function
 2793 // which returns its register number when queried.  CONST_INTER causes
 2794 // an operand to generate a function which returns the value of the
 2795 // constant when queried.  MEMORY_INTER causes an operand to generate
 2796 // four functions which return the Base Register, the Index Register,
 2797 // the Scale Value, and the Offset Value of the operand when queried.
 2798 // COND_INTER causes an operand to generate six functions which return
 2799 // the encoding code (ie - encoding bits for the instruction)
 2800 // associated with each basic boolean condition for a conditional
 2801 // instruction.
 2802 //
 2803 // Instructions specify two basic values for encoding.  Again, a
 2804 // function is available to check if the constant displacement is an
 2805 // oop. They use the ins_encode keyword to specify their encoding
 2806 // classes (which must be a sequence of enc_class names, and their
 2807 // parameters, specified in the encoding block), and they use the
 2808 // opcode keyword to specify, in order, their primary, secondary, and
 2809 // tertiary opcode.  Only the opcode sections which a particular
 2810 // instruction needs for encoding need to be specified.
 2811 encode %{
 2812   // Build emit functions for each basic byte or larger field in the
 2813   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2814   // from C++ code in the enc_class source block.  Emit functions will
 2815   // live in the main source block for now.  In future, we can
 2816   // generalize this by adding a syntax that specifies the sizes of
 2817   // fields in an order, so that the adlc can build the emit functions
 2818   // automagically
 2819 
 2820   // catch all for unimplemented encodings
 2821   enc_class enc_unimplemented %{
 2822     __ unimplemented("C2 catch all");
 2823   %}
 2824 
 2825   // BEGIN Non-volatile memory access
 2826 
 2827   // This encoding class is generated automatically from ad_encode.m4.
 2828   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2829   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2830     Register dst_reg = as_Register($dst$$reg);
 2831     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2832                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2833   %}
 2834 
 2835   // This encoding class is generated automatically from ad_encode.m4.
 2836   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2837   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2838     Register dst_reg = as_Register($dst$$reg);
 2839     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2840                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2841   %}
 2842 
 2843   // This encoding class is generated automatically from ad_encode.m4.
 2844   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2845   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2846     Register dst_reg = as_Register($dst$$reg);
 2847     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2848                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2849   %}
 2850 
 2851   // This encoding class is generated automatically from ad_encode.m4.
 2852   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2853   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2854     Register dst_reg = as_Register($dst$$reg);
 2855     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2856                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2857   %}
 2858 
 2859   // This encoding class is generated automatically from ad_encode.m4.
 2860   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2861   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2862     Register dst_reg = as_Register($dst$$reg);
 2863     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2864                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2865   %}
 2866 
 2867   // This encoding class is generated automatically from ad_encode.m4.
 2868   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2869   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2870     Register dst_reg = as_Register($dst$$reg);
 2871     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2872                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2873   %}
 2874 
 2875   // This encoding class is generated automatically from ad_encode.m4.
 2876   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2877   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2878     Register dst_reg = as_Register($dst$$reg);
 2879     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2880                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2881   %}
 2882 
 2883   // This encoding class is generated automatically from ad_encode.m4.
 2884   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2885   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2886     Register dst_reg = as_Register($dst$$reg);
 2887     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2888                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2889   %}
 2890 
 2891   // This encoding class is generated automatically from ad_encode.m4.
 2892   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2893   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2894     Register dst_reg = as_Register($dst$$reg);
 2895     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2896                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2897   %}
 2898 
 2899   // This encoding class is generated automatically from ad_encode.m4.
 2900   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2901   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2902     Register dst_reg = as_Register($dst$$reg);
 2903     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2904                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2905   %}
 2906 
 2907   // This encoding class is generated automatically from ad_encode.m4.
 2908   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2909   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2910     Register dst_reg = as_Register($dst$$reg);
 2911     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2912                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2913   %}
 2914 
 2915   // This encoding class is generated automatically from ad_encode.m4.
 2916   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2917   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2918     Register dst_reg = as_Register($dst$$reg);
 2919     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2920                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2921   %}
 2922 
 2923   // This encoding class is generated automatically from ad_encode.m4.
 2924   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2925   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2926     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2927     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2928                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2929   %}
 2930 
 2931   // This encoding class is generated automatically from ad_encode.m4.
 2932   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2933   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2934     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2935     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2936                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2937   %}
 2938 
 2939   // This encoding class is generated automatically from ad_encode.m4.
 2940   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2941   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2942     Register src_reg = as_Register($src$$reg);
 2943     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_strb0(memory1 mem) %{
 2950     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2957     Register src_reg = as_Register($src$$reg);
 2958     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_strh0(memory2 mem) %{
 2965     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2967   %}
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2972     Register src_reg = as_Register($src$$reg);
 2973     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_strw0(memory4 mem) %{
 2980     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2981                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2982   %}
 2983 
 2984   // This encoding class is generated automatically from ad_encode.m4.
 2985   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2986   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2987     Register src_reg = as_Register($src$$reg);
 2988     // we sometimes get asked to store the stack pointer into the
 2989     // current thread -- we cannot do that directly on AArch64
 2990     if (src_reg == r31_sp) {
 2991       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2992       __ mov(rscratch2, sp);
 2993       src_reg = rscratch2;
 2994     }
 2995     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 2996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2997   %}
 2998 
 2999   // This encoding class is generated automatically from ad_encode.m4.
 3000   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3001   enc_class aarch64_enc_str0(memory8 mem) %{
 3002     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3003                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3004   %}
 3005 
 3006   // This encoding class is generated automatically from ad_encode.m4.
 3007   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3008   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3009     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3010     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3011                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3012   %}
 3013 
 3014   // This encoding class is generated automatically from ad_encode.m4.
 3015   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3016   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3017     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3018     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3019                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3020   %}
 3021 
 3022   // This encoding class is generated automatically from ad_encode.m4.
 3023   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3024   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3025       __ membar(Assembler::StoreStore);
 3026       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3027                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3028   %}
 3029 
 3030   // END Non-volatile memory access
 3031 
 3032   // Vector loads and stores
 3033   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3034     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3035     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3036        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3037   %}
 3038 
 3039   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3040     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3041     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3042        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3043   %}
 3044 
 3045   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3046     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3047     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3048        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3049   %}
 3050 
 3051   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3052     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3053     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3054        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3055   %}
 3056 
 3057   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3058     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3059     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3060        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3061   %}
 3062 
 3063   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3064     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3065     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3066        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3067   %}
 3068 
 3069   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3070     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3071     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3072        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3073   %}
 3074 
 3075   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3076     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3077     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3078        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3079   %}
 3080 
 3081   // volatile loads and stores
 3082 
 3083   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3084     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3085                  rscratch1, stlrb);
 3086   %}
 3087 
 3088   enc_class aarch64_enc_stlrb0(memory mem) %{
 3089     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3090                  rscratch1, stlrb);
 3091   %}
 3092 
 3093   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3094     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3095                  rscratch1, stlrh);
 3096   %}
 3097 
 3098   enc_class aarch64_enc_stlrh0(memory mem) %{
 3099     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3100                  rscratch1, stlrh);
 3101   %}
 3102 
 3103   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3104     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3105                  rscratch1, stlrw);
 3106   %}
 3107 
 3108   enc_class aarch64_enc_stlrw0(memory mem) %{
 3109     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3110                  rscratch1, stlrw);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3114     Register dst_reg = as_Register($dst$$reg);
 3115     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3116              rscratch1, ldarb);
 3117     __ sxtbw(dst_reg, dst_reg);
 3118   %}
 3119 
 3120   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3121     Register dst_reg = as_Register($dst$$reg);
 3122     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3123              rscratch1, ldarb);
 3124     __ sxtb(dst_reg, dst_reg);
 3125   %}
 3126 
 3127   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3128     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3129              rscratch1, ldarb);
 3130   %}
 3131 
 3132   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3133     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3134              rscratch1, ldarb);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3138     Register dst_reg = as_Register($dst$$reg);
 3139     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3140              rscratch1, ldarh);
 3141     __ sxthw(dst_reg, dst_reg);
 3142   %}
 3143 
 3144   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3145     Register dst_reg = as_Register($dst$$reg);
 3146     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3147              rscratch1, ldarh);
 3148     __ sxth(dst_reg, dst_reg);
 3149   %}
 3150 
 3151   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3152     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3153              rscratch1, ldarh);
 3154   %}
 3155 
 3156   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3157     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3158              rscratch1, ldarh);
 3159   %}
 3160 
 3161   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3162     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3163              rscratch1, ldarw);
 3164   %}
 3165 
 3166   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3167     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3168              rscratch1, ldarw);
 3169   %}
 3170 
 3171   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3172     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3173              rscratch1, ldar);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3177     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3178              rscratch1, ldarw);
 3179     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3183     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184              rscratch1, ldar);
 3185     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3186   %}
 3187 
 3188   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3189     Register src_reg = as_Register($src$$reg);
 3190     // we sometimes get asked to store the stack pointer into the
 3191     // current thread -- we cannot do that directly on AArch64
 3192     if (src_reg == r31_sp) {
 3193       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3194       __ mov(rscratch2, sp);
 3195       src_reg = rscratch2;
 3196     }
 3197     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3198                  rscratch1, stlr);
 3199   %}
 3200 
 3201   enc_class aarch64_enc_stlr0(memory mem) %{
 3202     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3203                  rscratch1, stlr);
 3204   %}
 3205 
 3206   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3207     {
 3208       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3209       __ fmovs(rscratch2, src_reg);
 3210     }
 3211     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212                  rscratch1, stlrw);
 3213   %}
 3214 
 3215   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3216     {
 3217       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3218       __ fmovd(rscratch2, src_reg);
 3219     }
 3220     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3221                  rscratch1, stlr);
 3222   %}
 3223 
 3224   // synchronized read/update encodings
 3225 
 3226   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3227     Register dst_reg = as_Register($dst$$reg);
 3228     Register base = as_Register($mem$$base);
 3229     int index = $mem$$index;
 3230     int scale = $mem$$scale;
 3231     int disp = $mem$$disp;
 3232     if (index == -1) {
 3233        if (disp != 0) {
 3234         __ lea(rscratch1, Address(base, disp));
 3235         __ ldaxr(dst_reg, rscratch1);
 3236       } else {
 3237         // TODO
 3238         // should we ever get anything other than this case?
 3239         __ ldaxr(dst_reg, base);
 3240       }
 3241     } else {
 3242       Register index_reg = as_Register(index);
 3243       if (disp == 0) {
 3244         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3245         __ ldaxr(dst_reg, rscratch1);
 3246       } else {
 3247         __ lea(rscratch1, Address(base, disp));
 3248         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3249         __ ldaxr(dst_reg, rscratch1);
 3250       }
 3251     }
 3252   %}
 3253 
 3254   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3255     Register src_reg = as_Register($src$$reg);
 3256     Register base = as_Register($mem$$base);
 3257     int index = $mem$$index;
 3258     int scale = $mem$$scale;
 3259     int disp = $mem$$disp;
 3260     if (index == -1) {
 3261        if (disp != 0) {
 3262         __ lea(rscratch2, Address(base, disp));
 3263         __ stlxr(rscratch1, src_reg, rscratch2);
 3264       } else {
 3265         // TODO
 3266         // should we ever get anything other than this case?
 3267         __ stlxr(rscratch1, src_reg, base);
 3268       }
 3269     } else {
 3270       Register index_reg = as_Register(index);
 3271       if (disp == 0) {
 3272         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3273         __ stlxr(rscratch1, src_reg, rscratch2);
 3274       } else {
 3275         __ lea(rscratch2, Address(base, disp));
 3276         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3277         __ stlxr(rscratch1, src_reg, rscratch2);
 3278       }
 3279     }
 3280     __ cmpw(rscratch1, zr);
 3281   %}
 3282 
 3283   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3284     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3285     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3286                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3287                /*weak*/ false, noreg);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3291     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3292     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3293                Assembler::word, /*acquire*/ false, /*release*/ true,
 3294                /*weak*/ false, noreg);
 3295   %}
 3296 
 3297   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3298     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3299     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3300                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3301                /*weak*/ false, noreg);
 3302   %}
 3303 
 3304   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3305     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3306     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3307                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3308                /*weak*/ false, noreg);
 3309   %}
 3310 
 3311 
 3312   // The only difference between aarch64_enc_cmpxchg and
 3313   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3314   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3315   // lock.
 3316   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3325     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3326                Assembler::word, /*acquire*/ true, /*release*/ true,
 3327                /*weak*/ false, noreg);
 3328   %}
 3329 
 3330   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3331     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3332     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3333                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3334                /*weak*/ false, noreg);
 3335   %}
 3336 
 3337   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3338     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3339     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3340                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3341                /*weak*/ false, noreg);
 3342   %}
 3343 
 3344   // auxiliary used for CompareAndSwapX to set result register
 3345   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3346     Register res_reg = as_Register($res$$reg);
 3347     __ cset(res_reg, Assembler::EQ);
 3348   %}
 3349 
 3350   // prefetch encodings
 3351 
 3352   enc_class aarch64_enc_prefetchw(memory mem) %{
 3353     Register base = as_Register($mem$$base);
 3354     int index = $mem$$index;
 3355     int scale = $mem$$scale;
 3356     int disp = $mem$$disp;
 3357     if (index == -1) {
 3358       // Fix up any out-of-range offsets.
 3359       assert_different_registers(rscratch1, base);
 3360       Address addr = Address(base, disp);
 3361       addr = __ legitimize_address(addr, 8, rscratch1);
 3362       __ prfm(addr, PSTL1KEEP);
 3363     } else {
 3364       Register index_reg = as_Register(index);
 3365       if (disp == 0) {
 3366         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3367       } else {
 3368         __ lea(rscratch1, Address(base, disp));
 3369 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3370       }
 3371     }
 3372   %}
 3373 
 3374   // mov encodings
 3375 
 3376   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3377     uint32_t con = (uint32_t)$src$$constant;
 3378     Register dst_reg = as_Register($dst$$reg);
 3379     if (con == 0) {
 3380       __ movw(dst_reg, zr);
 3381     } else {
 3382       __ movw(dst_reg, con);
 3383     }
 3384   %}
 3385 
 3386   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3387     Register dst_reg = as_Register($dst$$reg);
 3388     uint64_t con = (uint64_t)$src$$constant;
 3389     if (con == 0) {
 3390       __ mov(dst_reg, zr);
 3391     } else {
 3392       __ mov(dst_reg, con);
 3393     }
 3394   %}
 3395 
 3396   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3397     Register dst_reg = as_Register($dst$$reg);
 3398     address con = (address)$src$$constant;
 3399     if (con == nullptr || con == (address)1) {
 3400       ShouldNotReachHere();
 3401     } else {
 3402       relocInfo::relocType rtype = $src->constant_reloc();
 3403       if (rtype == relocInfo::oop_type) {
 3404         __ movoop(dst_reg, (jobject)con);
 3405       } else if (rtype == relocInfo::metadata_type) {
 3406         __ mov_metadata(dst_reg, (Metadata*)con);
 3407       } else {
 3408         assert(rtype == relocInfo::none, "unexpected reloc type");
 3409         if (! __ is_valid_AArch64_address(con) ||
 3410             con < (address)(uintptr_t)os::vm_page_size()) {
 3411           __ mov(dst_reg, con);
 3412         } else {
 3413           uint64_t offset;
 3414           __ adrp(dst_reg, con, offset);
 3415           __ add(dst_reg, dst_reg, offset);
 3416         }
 3417       }
 3418     }
 3419   %}
 3420 
 3421   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3422     Register dst_reg = as_Register($dst$$reg);
 3423     __ mov(dst_reg, zr);
 3424   %}
 3425 
 3426   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3427     Register dst_reg = as_Register($dst$$reg);
 3428     __ mov(dst_reg, (uint64_t)1);
 3429   %}
 3430 
 3431   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3432     __ load_byte_map_base($dst$$Register);
 3433   %}
 3434 
 3435   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3436     Register dst_reg = as_Register($dst$$reg);
 3437     address con = (address)$src$$constant;
 3438     if (con == nullptr) {
 3439       ShouldNotReachHere();
 3440     } else {
 3441       relocInfo::relocType rtype = $src->constant_reloc();
 3442       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3443       __ set_narrow_oop(dst_reg, (jobject)con);
 3444     }
 3445   %}
 3446 
 3447   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3448     Register dst_reg = as_Register($dst$$reg);
 3449     __ mov(dst_reg, zr);
 3450   %}
 3451 
 3452   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3453     Register dst_reg = as_Register($dst$$reg);
 3454     address con = (address)$src$$constant;
 3455     if (con == nullptr) {
 3456       ShouldNotReachHere();
 3457     } else {
 3458       relocInfo::relocType rtype = $src->constant_reloc();
 3459       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3460       __ set_narrow_klass(dst_reg, (Klass *)con);
 3461     }
 3462   %}
 3463 
 3464   // arithmetic encodings
 3465 
 3466   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3467     Register dst_reg = as_Register($dst$$reg);
 3468     Register src_reg = as_Register($src1$$reg);
 3469     int32_t con = (int32_t)$src2$$constant;
 3470     // add has primary == 0, subtract has primary == 1
 3471     if ($primary) { con = -con; }
 3472     if (con < 0) {
 3473       __ subw(dst_reg, src_reg, -con);
 3474     } else {
 3475       __ addw(dst_reg, src_reg, con);
 3476     }
 3477   %}
 3478 
 3479   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     Register src_reg = as_Register($src1$$reg);
 3482     int32_t con = (int32_t)$src2$$constant;
 3483     // add has primary == 0, subtract has primary == 1
 3484     if ($primary) { con = -con; }
 3485     if (con < 0) {
 3486       __ sub(dst_reg, src_reg, -con);
 3487     } else {
 3488       __ add(dst_reg, src_reg, con);
 3489     }
 3490   %}
 3491 
 3492   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3493    Register dst_reg = as_Register($dst$$reg);
 3494    Register src1_reg = as_Register($src1$$reg);
 3495    Register src2_reg = as_Register($src2$$reg);
 3496     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3497   %}
 3498 
 3499   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3500    Register dst_reg = as_Register($dst$$reg);
 3501    Register src1_reg = as_Register($src1$$reg);
 3502    Register src2_reg = as_Register($src2$$reg);
 3503     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3504   %}
 3505 
 3506   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3507    Register dst_reg = as_Register($dst$$reg);
 3508    Register src1_reg = as_Register($src1$$reg);
 3509    Register src2_reg = as_Register($src2$$reg);
 3510     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3511   %}
 3512 
 3513   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3514    Register dst_reg = as_Register($dst$$reg);
 3515    Register src1_reg = as_Register($src1$$reg);
 3516    Register src2_reg = as_Register($src2$$reg);
 3517     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3518   %}
 3519 
 3520   // compare instruction encodings
 3521 
 3522   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3523     Register reg1 = as_Register($src1$$reg);
 3524     Register reg2 = as_Register($src2$$reg);
 3525     __ cmpw(reg1, reg2);
 3526   %}
 3527 
 3528   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3529     Register reg = as_Register($src1$$reg);
 3530     int32_t val = $src2$$constant;
 3531     if (val >= 0) {
 3532       __ subsw(zr, reg, val);
 3533     } else {
 3534       __ addsw(zr, reg, -val);
 3535     }
 3536   %}
 3537 
 3538   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3539     Register reg1 = as_Register($src1$$reg);
 3540     uint32_t val = (uint32_t)$src2$$constant;
 3541     __ movw(rscratch1, val);
 3542     __ cmpw(reg1, rscratch1);
 3543   %}
 3544 
 3545   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3546     Register reg1 = as_Register($src1$$reg);
 3547     Register reg2 = as_Register($src2$$reg);
 3548     __ cmp(reg1, reg2);
 3549   %}
 3550 
 3551   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3552     Register reg = as_Register($src1$$reg);
 3553     int64_t val = $src2$$constant;
 3554     if (val >= 0) {
 3555       __ subs(zr, reg, val);
 3556     } else if (val != -val) {
 3557       __ adds(zr, reg, -val);
 3558     } else {
 3559     // aargh, Long.MIN_VALUE is a special case
 3560       __ orr(rscratch1, zr, (uint64_t)val);
 3561       __ subs(zr, reg, rscratch1);
 3562     }
 3563   %}
 3564 
 3565   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3566     Register reg1 = as_Register($src1$$reg);
 3567     uint64_t val = (uint64_t)$src2$$constant;
 3568     __ mov(rscratch1, val);
 3569     __ cmp(reg1, rscratch1);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3573     Register reg1 = as_Register($src1$$reg);
 3574     Register reg2 = as_Register($src2$$reg);
 3575     __ cmp(reg1, reg2);
 3576   %}
 3577 
 3578   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3579     Register reg1 = as_Register($src1$$reg);
 3580     Register reg2 = as_Register($src2$$reg);
 3581     __ cmpw(reg1, reg2);
 3582   %}
 3583 
 3584   enc_class aarch64_enc_testp(iRegP src) %{
 3585     Register reg = as_Register($src$$reg);
 3586     __ cmp(reg, zr);
 3587   %}
 3588 
 3589   enc_class aarch64_enc_testn(iRegN src) %{
 3590     Register reg = as_Register($src$$reg);
 3591     __ cmpw(reg, zr);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_b(label lbl) %{
 3595     Label *L = $lbl$$label;
 3596     __ b(*L);
 3597   %}
 3598 
 3599   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3600     Label *L = $lbl$$label;
 3601     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3602   %}
 3603 
 3604   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3605     Label *L = $lbl$$label;
 3606     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3607   %}
 3608 
 3609   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3610   %{
 3611      Register sub_reg = as_Register($sub$$reg);
 3612      Register super_reg = as_Register($super$$reg);
 3613      Register temp_reg = as_Register($temp$$reg);
 3614      Register result_reg = as_Register($result$$reg);
 3615 
 3616      Label miss;
 3617      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3618                                      nullptr, &miss,
 3619                                      /*set_cond_codes:*/ true);
 3620      if ($primary) {
 3621        __ mov(result_reg, zr);
 3622      }
 3623      __ bind(miss);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_java_static_call(method meth) %{
 3627     address addr = (address)$meth$$method;
 3628     address call;
 3629     if (!_method) {
 3630       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3631       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3632       if (call == nullptr) {
 3633         ciEnv::current()->record_failure("CodeCache is full");
 3634         return;
 3635       }
 3636     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3637       // The NOP here is purely to ensure that eliding a call to
 3638       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3639       __ nop();
 3640       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3641     } else {
 3642       int method_index = resolved_method_index(masm);
 3643       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3644                                                   : static_call_Relocation::spec(method_index);
 3645       call = __ trampoline_call(Address(addr, rspec));
 3646       if (call == nullptr) {
 3647         ciEnv::current()->record_failure("CodeCache is full");
 3648         return;
 3649       }
 3650       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3651         // Calls of the same statically bound method can share
 3652         // a stub to the interpreter.
 3653         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3654       } else {
 3655         // Emit stub for static call
 3656         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3657         if (stub == nullptr) {
 3658           ciEnv::current()->record_failure("CodeCache is full");
 3659           return;
 3660         }
 3661       }
 3662     }
 3663 
 3664     __ post_call_nop();
 3665 
 3666     // Only non uncommon_trap calls need to reinitialize ptrue.
 3667     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3668       __ reinitialize_ptrue();
 3669     }
 3670   %}
 3671 
 3672   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3673     int method_index = resolved_method_index(masm);
 3674     address call = __ ic_call((address)$meth$$method, method_index);
 3675     if (call == nullptr) {
 3676       ciEnv::current()->record_failure("CodeCache is full");
 3677       return;
 3678     }
 3679     __ post_call_nop();
 3680     if (Compile::current()->max_vector_size() > 0) {
 3681       __ reinitialize_ptrue();
 3682     }
 3683   %}
 3684 
 3685   enc_class aarch64_enc_call_epilog() %{
 3686     if (VerifyStackAtCalls) {
 3687       // Check that stack depth is unchanged: find majik cookie on stack
 3688       __ call_Unimplemented();
 3689     }
 3690     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3691       // The last return value is not set by the callee but used to pass IsInit information to compiled code.
 3692       // Search for the corresponding projection, get the register and emit code that initialized it.
 3693       uint con = (tf()->range_cc()->cnt() - 1);
 3694       for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3695         ProjNode* proj = fast_out(i)->as_Proj();
 3696         if (proj->_con == con) {
 3697           // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
 3698           OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3699           VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3700           Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3701           __ cmp(r0, zr);
 3702           __ cset(toReg, Assembler::NE);
 3703           if (reg->is_stack()) {
 3704             int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3705             __ str(toReg, Address(sp, st_off));
 3706           }
 3707           break;
 3708         }
 3709       }
 3710       if (return_value_is_used()) {
 3711         // An inline type is returned as fields in multiple registers.
 3712         // R0 either contains an oop if the inline type is buffered or a pointer
 3713         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3714         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3715         // r0 &= (r0 & 1) - 1
 3716         __ andr(rscratch1, r0, 0x1);
 3717         __ sub(rscratch1, rscratch1, 0x1);
 3718         __ andr(r0, r0, rscratch1);
 3719       }
 3720     }
 3721   %}
 3722 
 3723   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3724     // some calls to generated routines (arraycopy code) are scheduled
 3725     // by C2 as runtime calls. if so we can call them using a br (they
 3726     // will be in a reachable segment) otherwise we have to use a blr
 3727     // which loads the absolute address into a register.
 3728     address entry = (address)$meth$$method;
 3729     CodeBlob *cb = CodeCache::find_blob(entry);
 3730     if (cb) {
 3731       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3732       if (call == nullptr) {
 3733         ciEnv::current()->record_failure("CodeCache is full");
 3734         return;
 3735       }
 3736       __ post_call_nop();
 3737     } else {
 3738       Label retaddr;
 3739       // Make the anchor frame walkable
 3740       __ adr(rscratch2, retaddr);
 3741       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3742       __ lea(rscratch1, RuntimeAddress(entry));
 3743       __ blr(rscratch1);
 3744       __ bind(retaddr);
 3745       __ post_call_nop();
 3746     }
 3747     if (Compile::current()->max_vector_size() > 0) {
 3748       __ reinitialize_ptrue();
 3749     }
 3750   %}
 3751 
 3752   enc_class aarch64_enc_rethrow() %{
 3753     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3754   %}
 3755 
 3756   enc_class aarch64_enc_ret() %{
 3757 #ifdef ASSERT
 3758     if (Compile::current()->max_vector_size() > 0) {
 3759       __ verify_ptrue();
 3760     }
 3761 #endif
 3762     __ ret(lr);
 3763   %}
 3764 
 3765   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3766     Register target_reg = as_Register($jump_target$$reg);
 3767     __ br(target_reg);
 3768   %}
 3769 
 3770   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3771     Register target_reg = as_Register($jump_target$$reg);
 3772     // exception oop should be in r0
 3773     // ret addr has been popped into lr
 3774     // callee expects it in r3
 3775     __ mov(r3, lr);
 3776     __ br(target_reg);
 3777   %}
 3778 
 3779 %}
 3780 
 3781 //----------FRAME--------------------------------------------------------------
 3782 // Definition of frame structure and management information.
 3783 //
 3784 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3785 //                             |   (to get allocators register number
 3786 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3787 //  r   CALLER     |        |
 3788 //  o     |        +--------+      pad to even-align allocators stack-slot
 3789 //  w     V        |  pad0  |        numbers; owned by CALLER
 3790 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3791 //  h     ^        |   in   |  5
 3792 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3793 //  |     |        |        |  3
 3794 //  |     |        +--------+
 3795 //  V     |        | old out|      Empty on Intel, window on Sparc
 3796 //        |    old |preserve|      Must be even aligned.
 3797 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3798 //        |        |   in   |  3   area for Intel ret address
 3799 //     Owned by    |preserve|      Empty on Sparc.
 3800 //       SELF      +--------+
 3801 //        |        |  pad2  |  2   pad to align old SP
 3802 //        |        +--------+  1
 3803 //        |        | locks  |  0
 3804 //        |        +--------+----> OptoReg::stack0(), even aligned
 3805 //        |        |  pad1  | 11   pad to align new SP
 3806 //        |        +--------+
 3807 //        |        |        | 10
 3808 //        |        | spills |  9   spills
 3809 //        V        |        |  8   (pad0 slot for callee)
 3810 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3811 //        ^        |  out   |  7
 3812 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3813 //     Owned by    +--------+
 3814 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3815 //        |    new |preserve|      Must be even-aligned.
 3816 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3817 //        |        |        |
 3818 //
 3819 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3820 //         known from SELF's arguments and the Java calling convention.
 3821 //         Region 6-7 is determined per call site.
 3822 // Note 2: If the calling convention leaves holes in the incoming argument
 3823 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3824 //         are owned by the CALLEE.  Holes should not be necessary in the
 3825 //         incoming area, as the Java calling convention is completely under
 3826 //         the control of the AD file.  Doubles can be sorted and packed to
 3827 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3828 //         varargs C calling conventions.
 3829 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3830 //         even aligned with pad0 as needed.
 3831 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3832 //           (the latter is true on Intel but is it false on AArch64?)
 3833 //         region 6-11 is even aligned; it may be padded out more so that
 3834 //         the region from SP to FP meets the minimum stack alignment.
 3835 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3836 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3837 //         SP meets the minimum alignment.
 3838 
 3839 frame %{
 3840   // These three registers define part of the calling convention
 3841   // between compiled code and the interpreter.
 3842 
 3843   // Inline Cache Register or Method for I2C.
 3844   inline_cache_reg(R12);
 3845 
 3846   // Number of stack slots consumed by locking an object
 3847   sync_stack_slots(2);
 3848 
 3849   // Compiled code's Frame Pointer
 3850   frame_pointer(R31);
 3851 
 3852   // Interpreter stores its frame pointer in a register which is
 3853   // stored to the stack by I2CAdaptors.
 3854   // I2CAdaptors convert from interpreted java to compiled java.
 3855   interpreter_frame_pointer(R29);
 3856 
 3857   // Stack alignment requirement
 3858   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3859 
 3860   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3861   // for calls to C.  Supports the var-args backing area for register parms.
 3862   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3863 
 3864   // The after-PROLOG location of the return address.  Location of
 3865   // return address specifies a type (REG or STACK) and a number
 3866   // representing the register number (i.e. - use a register name) or
 3867   // stack slot.
 3868   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3869   // Otherwise, it is above the locks and verification slot and alignment word
 3870   // TODO this may well be correct but need to check why that - 2 is there
 3871   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3872   // which folds in the space used for monitors
 3873   return_addr(STACK - 2 +
 3874               align_up((Compile::current()->in_preserve_stack_slots() +
 3875                         Compile::current()->fixed_slots()),
 3876                        stack_alignment_in_slots()));
 3877 
 3878   // Location of compiled Java return values.  Same as C for now.
 3879   return_value
 3880   %{
 3881     // TODO do we allow ideal_reg == Op_RegN???
 3882     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3883            "only return normal values");
 3884 
 3885     static const int lo[Op_RegL + 1] = { // enum name
 3886       0,                                 // Op_Node
 3887       0,                                 // Op_Set
 3888       R0_num,                            // Op_RegN
 3889       R0_num,                            // Op_RegI
 3890       R0_num,                            // Op_RegP
 3891       V0_num,                            // Op_RegF
 3892       V0_num,                            // Op_RegD
 3893       R0_num                             // Op_RegL
 3894     };
 3895 
 3896     static const int hi[Op_RegL + 1] = { // enum name
 3897       0,                                 // Op_Node
 3898       0,                                 // Op_Set
 3899       OptoReg::Bad,                      // Op_RegN
 3900       OptoReg::Bad,                      // Op_RegI
 3901       R0_H_num,                          // Op_RegP
 3902       OptoReg::Bad,                      // Op_RegF
 3903       V0_H_num,                          // Op_RegD
 3904       R0_H_num                           // Op_RegL
 3905     };
 3906 
 3907     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3908   %}
 3909 %}
 3910 
 3911 //----------ATTRIBUTES---------------------------------------------------------
 3912 //----------Operand Attributes-------------------------------------------------
 3913 op_attrib op_cost(1);        // Required cost attribute
 3914 
 3915 //----------Instruction Attributes---------------------------------------------
 3916 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3917 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3918 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3919                                 // a non-matching short branch variant
 3920                                 // of some long branch?
 3921 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3922                                 // be a power of 2) specifies the
 3923                                 // alignment that some part of the
 3924                                 // instruction (not necessarily the
 3925                                 // start) requires.  If > 1, a
 3926                                 // compute_padding() function must be
 3927                                 // provided for the instruction
 3928 
 3929 //----------OPERANDS-----------------------------------------------------------
 3930 // Operand definitions must precede instruction definitions for correct parsing
 3931 // in the ADLC because operands constitute user defined types which are used in
 3932 // instruction definitions.
 3933 
 3934 //----------Simple Operands----------------------------------------------------
 3935 
 3936 // Integer operands 32 bit
 3937 // 32 bit immediate
 3938 operand immI()
 3939 %{
 3940   match(ConI);
 3941 
 3942   op_cost(0);
 3943   format %{ %}
 3944   interface(CONST_INTER);
 3945 %}
 3946 
 3947 // 32 bit zero
 3948 operand immI0()
 3949 %{
 3950   predicate(n->get_int() == 0);
 3951   match(ConI);
 3952 
 3953   op_cost(0);
 3954   format %{ %}
 3955   interface(CONST_INTER);
 3956 %}
 3957 
 3958 // 32 bit unit increment
 3959 operand immI_1()
 3960 %{
 3961   predicate(n->get_int() == 1);
 3962   match(ConI);
 3963 
 3964   op_cost(0);
 3965   format %{ %}
 3966   interface(CONST_INTER);
 3967 %}
 3968 
 3969 // 32 bit unit decrement
 3970 operand immI_M1()
 3971 %{
 3972   predicate(n->get_int() == -1);
 3973   match(ConI);
 3974 
 3975   op_cost(0);
 3976   format %{ %}
 3977   interface(CONST_INTER);
 3978 %}
 3979 
 3980 // Shift values for add/sub extension shift
 3981 operand immIExt()
 3982 %{
 3983   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 3984   match(ConI);
 3985 
 3986   op_cost(0);
 3987   format %{ %}
 3988   interface(CONST_INTER);
 3989 %}
 3990 
 3991 operand immI_gt_1()
 3992 %{
 3993   predicate(n->get_int() > 1);
 3994   match(ConI);
 3995 
 3996   op_cost(0);
 3997   format %{ %}
 3998   interface(CONST_INTER);
 3999 %}
 4000 
 4001 operand immI_le_4()
 4002 %{
 4003   predicate(n->get_int() <= 4);
 4004   match(ConI);
 4005 
 4006   op_cost(0);
 4007   format %{ %}
 4008   interface(CONST_INTER);
 4009 %}
 4010 
 4011 operand immI_16()
 4012 %{
 4013   predicate(n->get_int() == 16);
 4014   match(ConI);
 4015 
 4016   op_cost(0);
 4017   format %{ %}
 4018   interface(CONST_INTER);
 4019 %}
 4020 
 4021 operand immI_24()
 4022 %{
 4023   predicate(n->get_int() == 24);
 4024   match(ConI);
 4025 
 4026   op_cost(0);
 4027   format %{ %}
 4028   interface(CONST_INTER);
 4029 %}
 4030 
 4031 operand immI_32()
 4032 %{
 4033   predicate(n->get_int() == 32);
 4034   match(ConI);
 4035 
 4036   op_cost(0);
 4037   format %{ %}
 4038   interface(CONST_INTER);
 4039 %}
 4040 
 4041 operand immI_48()
 4042 %{
 4043   predicate(n->get_int() == 48);
 4044   match(ConI);
 4045 
 4046   op_cost(0);
 4047   format %{ %}
 4048   interface(CONST_INTER);
 4049 %}
 4050 
 4051 operand immI_56()
 4052 %{
 4053   predicate(n->get_int() == 56);
 4054   match(ConI);
 4055 
 4056   op_cost(0);
 4057   format %{ %}
 4058   interface(CONST_INTER);
 4059 %}
 4060 
 4061 operand immI_255()
 4062 %{
 4063   predicate(n->get_int() == 255);
 4064   match(ConI);
 4065 
 4066   op_cost(0);
 4067   format %{ %}
 4068   interface(CONST_INTER);
 4069 %}
 4070 
 4071 operand immI_65535()
 4072 %{
 4073   predicate(n->get_int() == 65535);
 4074   match(ConI);
 4075 
 4076   op_cost(0);
 4077   format %{ %}
 4078   interface(CONST_INTER);
 4079 %}
 4080 
 4081 operand immI_positive()
 4082 %{
 4083   predicate(n->get_int() > 0);
 4084   match(ConI);
 4085 
 4086   op_cost(0);
 4087   format %{ %}
 4088   interface(CONST_INTER);
 4089 %}
 4090 
 4091 // BoolTest condition for signed compare
 4092 operand immI_cmp_cond()
 4093 %{
 4094   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4095   match(ConI);
 4096 
 4097   op_cost(0);
 4098   format %{ %}
 4099   interface(CONST_INTER);
 4100 %}
 4101 
 4102 // BoolTest condition for unsigned compare
 4103 operand immI_cmpU_cond()
 4104 %{
 4105   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4106   match(ConI);
 4107 
 4108   op_cost(0);
 4109   format %{ %}
 4110   interface(CONST_INTER);
 4111 %}
 4112 
 4113 operand immL_255()
 4114 %{
 4115   predicate(n->get_long() == 255L);
 4116   match(ConL);
 4117 
 4118   op_cost(0);
 4119   format %{ %}
 4120   interface(CONST_INTER);
 4121 %}
 4122 
 4123 operand immL_65535()
 4124 %{
 4125   predicate(n->get_long() == 65535L);
 4126   match(ConL);
 4127 
 4128   op_cost(0);
 4129   format %{ %}
 4130   interface(CONST_INTER);
 4131 %}
 4132 
 4133 operand immL_4294967295()
 4134 %{
 4135   predicate(n->get_long() == 4294967295L);
 4136   match(ConL);
 4137 
 4138   op_cost(0);
 4139   format %{ %}
 4140   interface(CONST_INTER);
 4141 %}
 4142 
 4143 operand immL_bitmask()
 4144 %{
 4145   predicate((n->get_long() != 0)
 4146             && ((n->get_long() & 0xc000000000000000l) == 0)
 4147             && is_power_of_2(n->get_long() + 1));
 4148   match(ConL);
 4149 
 4150   op_cost(0);
 4151   format %{ %}
 4152   interface(CONST_INTER);
 4153 %}
 4154 
 4155 operand immI_bitmask()
 4156 %{
 4157   predicate((n->get_int() != 0)
 4158             && ((n->get_int() & 0xc0000000) == 0)
 4159             && is_power_of_2(n->get_int() + 1));
 4160   match(ConI);
 4161 
 4162   op_cost(0);
 4163   format %{ %}
 4164   interface(CONST_INTER);
 4165 %}
 4166 
 4167 operand immL_positive_bitmaskI()
 4168 %{
 4169   predicate((n->get_long() != 0)
 4170             && ((julong)n->get_long() < 0x80000000ULL)
 4171             && is_power_of_2(n->get_long() + 1));
 4172   match(ConL);
 4173 
 4174   op_cost(0);
 4175   format %{ %}
 4176   interface(CONST_INTER);
 4177 %}
 4178 
 4179 // Scale values for scaled offset addressing modes (up to long but not quad)
 4180 operand immIScale()
 4181 %{
 4182   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4183   match(ConI);
 4184 
 4185   op_cost(0);
 4186   format %{ %}
 4187   interface(CONST_INTER);
 4188 %}
 4189 
 4190 // 5 bit signed integer
 4191 operand immI5()
 4192 %{
 4193   predicate(Assembler::is_simm(n->get_int(), 5));
 4194   match(ConI);
 4195 
 4196   op_cost(0);
 4197   format %{ %}
 4198   interface(CONST_INTER);
 4199 %}
 4200 
 4201 // 7 bit unsigned integer
 4202 operand immIU7()
 4203 %{
 4204   predicate(Assembler::is_uimm(n->get_int(), 7));
 4205   match(ConI);
 4206 
 4207   op_cost(0);
 4208   format %{ %}
 4209   interface(CONST_INTER);
 4210 %}
 4211 
 4212 // Offset for scaled or unscaled immediate loads and stores
 4213 operand immIOffset()
 4214 %{
 4215   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4216   match(ConI);
 4217 
 4218   op_cost(0);
 4219   format %{ %}
 4220   interface(CONST_INTER);
 4221 %}
 4222 
 4223 operand immIOffset1()
 4224 %{
 4225   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4226   match(ConI);
 4227 
 4228   op_cost(0);
 4229   format %{ %}
 4230   interface(CONST_INTER);
 4231 %}
 4232 
 4233 operand immIOffset2()
 4234 %{
 4235   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4236   match(ConI);
 4237 
 4238   op_cost(0);
 4239   format %{ %}
 4240   interface(CONST_INTER);
 4241 %}
 4242 
 4243 operand immIOffset4()
 4244 %{
 4245   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4246   match(ConI);
 4247 
 4248   op_cost(0);
 4249   format %{ %}
 4250   interface(CONST_INTER);
 4251 %}
 4252 
 4253 operand immIOffset8()
 4254 %{
 4255   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4256   match(ConI);
 4257 
 4258   op_cost(0);
 4259   format %{ %}
 4260   interface(CONST_INTER);
 4261 %}
 4262 
 4263 operand immIOffset16()
 4264 %{
 4265   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4266   match(ConI);
 4267 
 4268   op_cost(0);
 4269   format %{ %}
 4270   interface(CONST_INTER);
 4271 %}
 4272 
 4273 operand immLOffset()
 4274 %{
 4275   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4276   match(ConL);
 4277 
 4278   op_cost(0);
 4279   format %{ %}
 4280   interface(CONST_INTER);
 4281 %}
 4282 
 4283 operand immLoffset1()
 4284 %{
 4285   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4286   match(ConL);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immLoffset2()
 4294 %{
 4295   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4296   match(ConL);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 operand immLoffset4()
 4304 %{
 4305   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4306   match(ConL);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 operand immLoffset8()
 4314 %{
 4315   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4316   match(ConL);
 4317 
 4318   op_cost(0);
 4319   format %{ %}
 4320   interface(CONST_INTER);
 4321 %}
 4322 
 4323 operand immLoffset16()
 4324 %{
 4325   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4326   match(ConL);
 4327 
 4328   op_cost(0);
 4329   format %{ %}
 4330   interface(CONST_INTER);
 4331 %}
 4332 
 4333 // 5 bit signed long integer
 4334 operand immL5()
 4335 %{
 4336   predicate(Assembler::is_simm(n->get_long(), 5));
 4337   match(ConL);
 4338 
 4339   op_cost(0);
 4340   format %{ %}
 4341   interface(CONST_INTER);
 4342 %}
 4343 
 4344 // 7 bit unsigned long integer
 4345 operand immLU7()
 4346 %{
 4347   predicate(Assembler::is_uimm(n->get_long(), 7));
 4348   match(ConL);
 4349 
 4350   op_cost(0);
 4351   format %{ %}
 4352   interface(CONST_INTER);
 4353 %}
 4354 
 4355 // 8 bit signed value.
 4356 operand immI8()
 4357 %{
 4358   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4359   match(ConI);
 4360 
 4361   op_cost(0);
 4362   format %{ %}
 4363   interface(CONST_INTER);
 4364 %}
 4365 
 4366 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4367 operand immI8_shift8()
 4368 %{
 4369   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4370             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4371   match(ConI);
 4372 
 4373   op_cost(0);
 4374   format %{ %}
 4375   interface(CONST_INTER);
 4376 %}
 4377 
 4378 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4379 operand immL8_shift8()
 4380 %{
 4381   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4382             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4383   match(ConL);
 4384 
 4385   op_cost(0);
 4386   format %{ %}
 4387   interface(CONST_INTER);
 4388 %}
 4389 
 4390 // 8 bit integer valid for vector add sub immediate
 4391 operand immBAddSubV()
 4392 %{
 4393   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 // 32 bit integer valid for add sub immediate
 4402 operand immIAddSub()
 4403 %{
 4404   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4405   match(ConI);
 4406   op_cost(0);
 4407   format %{ %}
 4408   interface(CONST_INTER);
 4409 %}
 4410 
 4411 // 32 bit integer valid for vector add sub immediate
 4412 operand immIAddSubV()
 4413 %{
 4414   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4415   match(ConI);
 4416 
 4417   op_cost(0);
 4418   format %{ %}
 4419   interface(CONST_INTER);
 4420 %}
 4421 
 4422 // 32 bit unsigned integer valid for logical immediate
 4423 
 4424 operand immBLog()
 4425 %{
 4426   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4427   match(ConI);
 4428 
 4429   op_cost(0);
 4430   format %{ %}
 4431   interface(CONST_INTER);
 4432 %}
 4433 
 4434 operand immSLog()
 4435 %{
 4436   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4437   match(ConI);
 4438 
 4439   op_cost(0);
 4440   format %{ %}
 4441   interface(CONST_INTER);
 4442 %}
 4443 
 4444 operand immILog()
 4445 %{
 4446   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4447   match(ConI);
 4448 
 4449   op_cost(0);
 4450   format %{ %}
 4451   interface(CONST_INTER);
 4452 %}
 4453 
 4454 // Integer operands 64 bit
 4455 // 64 bit immediate
 4456 operand immL()
 4457 %{
 4458   match(ConL);
 4459 
 4460   op_cost(0);
 4461   format %{ %}
 4462   interface(CONST_INTER);
 4463 %}
 4464 
 4465 // 64 bit zero
 4466 operand immL0()
 4467 %{
 4468   predicate(n->get_long() == 0);
 4469   match(ConL);
 4470 
 4471   op_cost(0);
 4472   format %{ %}
 4473   interface(CONST_INTER);
 4474 %}
 4475 
 4476 // 64 bit unit decrement
 4477 operand immL_M1()
 4478 %{
 4479   predicate(n->get_long() == -1);
 4480   match(ConL);
 4481 
 4482   op_cost(0);
 4483   format %{ %}
 4484   interface(CONST_INTER);
 4485 %}
 4486 
 4487 // 64 bit integer valid for add sub immediate
 4488 operand immLAddSub()
 4489 %{
 4490   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4491   match(ConL);
 4492   op_cost(0);
 4493   format %{ %}
 4494   interface(CONST_INTER);
 4495 %}
 4496 
 4497 // 64 bit integer valid for addv subv immediate
 4498 operand immLAddSubV()
 4499 %{
 4500   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4501   match(ConL);
 4502 
 4503   op_cost(0);
 4504   format %{ %}
 4505   interface(CONST_INTER);
 4506 %}
 4507 
 4508 // 64 bit integer valid for logical immediate
 4509 operand immLLog()
 4510 %{
 4511   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4512   match(ConL);
 4513   op_cost(0);
 4514   format %{ %}
 4515   interface(CONST_INTER);
 4516 %}
 4517 
 4518 // Long Immediate: low 32-bit mask
 4519 operand immL_32bits()
 4520 %{
 4521   predicate(n->get_long() == 0xFFFFFFFFL);
 4522   match(ConL);
 4523   op_cost(0);
 4524   format %{ %}
 4525   interface(CONST_INTER);
 4526 %}
 4527 
 4528 // Pointer operands
 4529 // Pointer Immediate
 4530 operand immP()
 4531 %{
 4532   match(ConP);
 4533 
 4534   op_cost(0);
 4535   format %{ %}
 4536   interface(CONST_INTER);
 4537 %}
 4538 
 4539 // nullptr Pointer Immediate
 4540 operand immP0()
 4541 %{
 4542   predicate(n->get_ptr() == 0);
 4543   match(ConP);
 4544 
 4545   op_cost(0);
 4546   format %{ %}
 4547   interface(CONST_INTER);
 4548 %}
 4549 
 4550 // Pointer Immediate One
 4551 // this is used in object initialization (initial object header)
 4552 operand immP_1()
 4553 %{
 4554   predicate(n->get_ptr() == 1);
 4555   match(ConP);
 4556 
 4557   op_cost(0);
 4558   format %{ %}
 4559   interface(CONST_INTER);
 4560 %}
 4561 
 4562 // Card Table Byte Map Base
 4563 operand immByteMapBase()
 4564 %{
 4565   // Get base of card map
 4566   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4567             SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
 4568             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4569   match(ConP);
 4570 
 4571   op_cost(0);
 4572   format %{ %}
 4573   interface(CONST_INTER);
 4574 %}
 4575 
 4576 // Float and Double operands
 4577 // Double Immediate
 4578 operand immD()
 4579 %{
 4580   match(ConD);
 4581   op_cost(0);
 4582   format %{ %}
 4583   interface(CONST_INTER);
 4584 %}
 4585 
 4586 // Double Immediate: +0.0d
 4587 operand immD0()
 4588 %{
 4589   predicate(jlong_cast(n->getd()) == 0);
 4590   match(ConD);
 4591 
 4592   op_cost(0);
 4593   format %{ %}
 4594   interface(CONST_INTER);
 4595 %}
 4596 
 4597 // constant 'double +0.0'.
 4598 operand immDPacked()
 4599 %{
 4600   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4601   match(ConD);
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // Float Immediate
 4608 operand immF()
 4609 %{
 4610   match(ConF);
 4611   op_cost(0);
 4612   format %{ %}
 4613   interface(CONST_INTER);
 4614 %}
 4615 
 4616 // Float Immediate: +0.0f.
 4617 operand immF0()
 4618 %{
 4619   predicate(jint_cast(n->getf()) == 0);
 4620   match(ConF);
 4621 
 4622   op_cost(0);
 4623   format %{ %}
 4624   interface(CONST_INTER);
 4625 %}
 4626 
 4627 //
 4628 operand immFPacked()
 4629 %{
 4630   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4631   match(ConF);
 4632   op_cost(0);
 4633   format %{ %}
 4634   interface(CONST_INTER);
 4635 %}
 4636 
 4637 // Narrow pointer operands
 4638 // Narrow Pointer Immediate
 4639 operand immN()
 4640 %{
 4641   match(ConN);
 4642 
 4643   op_cost(0);
 4644   format %{ %}
 4645   interface(CONST_INTER);
 4646 %}
 4647 
 4648 // Narrow nullptr Pointer Immediate
 4649 operand immN0()
 4650 %{
 4651   predicate(n->get_narrowcon() == 0);
 4652   match(ConN);
 4653 
 4654   op_cost(0);
 4655   format %{ %}
 4656   interface(CONST_INTER);
 4657 %}
 4658 
 4659 operand immNKlass()
 4660 %{
 4661   match(ConNKlass);
 4662 
 4663   op_cost(0);
 4664   format %{ %}
 4665   interface(CONST_INTER);
 4666 %}
 4667 
 4668 // Integer 32 bit Register Operands
 4669 // Integer 32 bitRegister (excludes SP)
 4670 operand iRegI()
 4671 %{
 4672   constraint(ALLOC_IN_RC(any_reg32));
 4673   match(RegI);
 4674   match(iRegINoSp);
 4675   op_cost(0);
 4676   format %{ %}
 4677   interface(REG_INTER);
 4678 %}
 4679 
 4680 // Integer 32 bit Register not Special
 4681 operand iRegINoSp()
 4682 %{
 4683   constraint(ALLOC_IN_RC(no_special_reg32));
 4684   match(RegI);
 4685   op_cost(0);
 4686   format %{ %}
 4687   interface(REG_INTER);
 4688 %}
 4689 
 4690 // Integer 64 bit Register Operands
 4691 // Integer 64 bit Register (includes SP)
 4692 operand iRegL()
 4693 %{
 4694   constraint(ALLOC_IN_RC(any_reg));
 4695   match(RegL);
 4696   match(iRegLNoSp);
 4697   op_cost(0);
 4698   format %{ %}
 4699   interface(REG_INTER);
 4700 %}
 4701 
 4702 // Integer 64 bit Register not Special
 4703 operand iRegLNoSp()
 4704 %{
 4705   constraint(ALLOC_IN_RC(no_special_reg));
 4706   match(RegL);
 4707   match(iRegL_R0);
 4708   format %{ %}
 4709   interface(REG_INTER);
 4710 %}
 4711 
 4712 // Pointer Register Operands
 4713 // Pointer Register
 4714 operand iRegP()
 4715 %{
 4716   constraint(ALLOC_IN_RC(ptr_reg));
 4717   match(RegP);
 4718   match(iRegPNoSp);
 4719   match(iRegP_R0);
 4720   //match(iRegP_R2);
 4721   //match(iRegP_R4);
 4722   match(iRegP_R5);
 4723   match(thread_RegP);
 4724   op_cost(0);
 4725   format %{ %}
 4726   interface(REG_INTER);
 4727 %}
 4728 
 4729 // Pointer 64 bit Register not Special
 4730 operand iRegPNoSp()
 4731 %{
 4732   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4733   match(RegP);
 4734   // match(iRegP);
 4735   // match(iRegP_R0);
 4736   // match(iRegP_R2);
 4737   // match(iRegP_R4);
 4738   // match(iRegP_R5);
 4739   // match(thread_RegP);
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(REG_INTER);
 4743 %}
 4744 
 4745 // This operand is not allowed to use rfp even if
 4746 // rfp is not used to hold the frame pointer.
 4747 operand iRegPNoSpNoRfp()
 4748 %{
 4749   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4750   match(RegP);
 4751   match(iRegPNoSp);
 4752   op_cost(0);
 4753   format %{ %}
 4754   interface(REG_INTER);
 4755 %}
 4756 
 4757 // Pointer 64 bit Register R0 only
 4758 operand iRegP_R0()
 4759 %{
 4760   constraint(ALLOC_IN_RC(r0_reg));
 4761   match(RegP);
 4762   // match(iRegP);
 4763   match(iRegPNoSp);
 4764   op_cost(0);
 4765   format %{ %}
 4766   interface(REG_INTER);
 4767 %}
 4768 
 4769 // Pointer 64 bit Register R1 only
 4770 operand iRegP_R1()
 4771 %{
 4772   constraint(ALLOC_IN_RC(r1_reg));
 4773   match(RegP);
 4774   // match(iRegP);
 4775   match(iRegPNoSp);
 4776   op_cost(0);
 4777   format %{ %}
 4778   interface(REG_INTER);
 4779 %}
 4780 
 4781 // Pointer 64 bit Register R2 only
 4782 operand iRegP_R2()
 4783 %{
 4784   constraint(ALLOC_IN_RC(r2_reg));
 4785   match(RegP);
 4786   // match(iRegP);
 4787   match(iRegPNoSp);
 4788   op_cost(0);
 4789   format %{ %}
 4790   interface(REG_INTER);
 4791 %}
 4792 
 4793 // Pointer 64 bit Register R3 only
 4794 operand iRegP_R3()
 4795 %{
 4796   constraint(ALLOC_IN_RC(r3_reg));
 4797   match(RegP);
 4798   // match(iRegP);
 4799   match(iRegPNoSp);
 4800   op_cost(0);
 4801   format %{ %}
 4802   interface(REG_INTER);
 4803 %}
 4804 
 4805 // Pointer 64 bit Register R4 only
 4806 operand iRegP_R4()
 4807 %{
 4808   constraint(ALLOC_IN_RC(r4_reg));
 4809   match(RegP);
 4810   // match(iRegP);
 4811   match(iRegPNoSp);
 4812   op_cost(0);
 4813   format %{ %}
 4814   interface(REG_INTER);
 4815 %}
 4816 
 4817 // Pointer 64 bit Register R5 only
 4818 operand iRegP_R5()
 4819 %{
 4820   constraint(ALLOC_IN_RC(r5_reg));
 4821   match(RegP);
 4822   // match(iRegP);
 4823   match(iRegPNoSp);
 4824   op_cost(0);
 4825   format %{ %}
 4826   interface(REG_INTER);
 4827 %}
 4828 
 4829 // Pointer 64 bit Register R10 only
 4830 operand iRegP_R10()
 4831 %{
 4832   constraint(ALLOC_IN_RC(r10_reg));
 4833   match(RegP);
 4834   // match(iRegP);
 4835   match(iRegPNoSp);
 4836   op_cost(0);
 4837   format %{ %}
 4838   interface(REG_INTER);
 4839 %}
 4840 
 4841 // Long 64 bit Register R0 only
 4842 operand iRegL_R0()
 4843 %{
 4844   constraint(ALLOC_IN_RC(r0_reg));
 4845   match(RegL);
 4846   match(iRegLNoSp);
 4847   op_cost(0);
 4848   format %{ %}
 4849   interface(REG_INTER);
 4850 %}
 4851 
 4852 // Long 64 bit Register R11 only
 4853 operand iRegL_R11()
 4854 %{
 4855   constraint(ALLOC_IN_RC(r11_reg));
 4856   match(RegL);
 4857   match(iRegLNoSp);
 4858   op_cost(0);
 4859   format %{ %}
 4860   interface(REG_INTER);
 4861 %}
 4862 
 4863 // Register R0 only
 4864 operand iRegI_R0()
 4865 %{
 4866   constraint(ALLOC_IN_RC(int_r0_reg));
 4867   match(RegI);
 4868   match(iRegINoSp);
 4869   op_cost(0);
 4870   format %{ %}
 4871   interface(REG_INTER);
 4872 %}
 4873 
 4874 // Register R2 only
 4875 operand iRegI_R2()
 4876 %{
 4877   constraint(ALLOC_IN_RC(int_r2_reg));
 4878   match(RegI);
 4879   match(iRegINoSp);
 4880   op_cost(0);
 4881   format %{ %}
 4882   interface(REG_INTER);
 4883 %}
 4884 
 4885 // Register R3 only
 4886 operand iRegI_R3()
 4887 %{
 4888   constraint(ALLOC_IN_RC(int_r3_reg));
 4889   match(RegI);
 4890   match(iRegINoSp);
 4891   op_cost(0);
 4892   format %{ %}
 4893   interface(REG_INTER);
 4894 %}
 4895 
 4896 
 4897 // Register R4 only
 4898 operand iRegI_R4()
 4899 %{
 4900   constraint(ALLOC_IN_RC(int_r4_reg));
 4901   match(RegI);
 4902   match(iRegINoSp);
 4903   op_cost(0);
 4904   format %{ %}
 4905   interface(REG_INTER);
 4906 %}
 4907 
 4908 
 4909 // Pointer Register Operands
 4910 // Narrow Pointer Register
 4911 operand iRegN()
 4912 %{
 4913   constraint(ALLOC_IN_RC(any_reg32));
 4914   match(RegN);
 4915   match(iRegNNoSp);
 4916   op_cost(0);
 4917   format %{ %}
 4918   interface(REG_INTER);
 4919 %}
 4920 
 4921 // Integer 64 bit Register not Special
 4922 operand iRegNNoSp()
 4923 %{
 4924   constraint(ALLOC_IN_RC(no_special_reg32));
 4925   match(RegN);
 4926   op_cost(0);
 4927   format %{ %}
 4928   interface(REG_INTER);
 4929 %}
 4930 
 4931 // Float Register
 4932 // Float register operands
 4933 operand vRegF()
 4934 %{
 4935   constraint(ALLOC_IN_RC(float_reg));
 4936   match(RegF);
 4937 
 4938   op_cost(0);
 4939   format %{ %}
 4940   interface(REG_INTER);
 4941 %}
 4942 
 4943 // Double Register
 4944 // Double register operands
 4945 operand vRegD()
 4946 %{
 4947   constraint(ALLOC_IN_RC(double_reg));
 4948   match(RegD);
 4949 
 4950   op_cost(0);
 4951   format %{ %}
 4952   interface(REG_INTER);
 4953 %}
 4954 
 4955 // Generic vector class. This will be used for
 4956 // all vector operands, including NEON and SVE.
 4957 operand vReg()
 4958 %{
 4959   constraint(ALLOC_IN_RC(dynamic));
 4960   match(VecA);
 4961   match(VecD);
 4962   match(VecX);
 4963 
 4964   op_cost(0);
 4965   format %{ %}
 4966   interface(REG_INTER);
 4967 %}
 4968 
 4969 operand vecA()
 4970 %{
 4971   constraint(ALLOC_IN_RC(vectora_reg));
 4972   match(VecA);
 4973 
 4974   op_cost(0);
 4975   format %{ %}
 4976   interface(REG_INTER);
 4977 %}
 4978 
 4979 operand vecD()
 4980 %{
 4981   constraint(ALLOC_IN_RC(vectord_reg));
 4982   match(VecD);
 4983 
 4984   op_cost(0);
 4985   format %{ %}
 4986   interface(REG_INTER);
 4987 %}
 4988 
 4989 operand vecX()
 4990 %{
 4991   constraint(ALLOC_IN_RC(vectorx_reg));
 4992   match(VecX);
 4993 
 4994   op_cost(0);
 4995   format %{ %}
 4996   interface(REG_INTER);
 4997 %}
 4998 
 4999 operand vRegD_V0()
 5000 %{
 5001   constraint(ALLOC_IN_RC(v0_reg));
 5002   match(RegD);
 5003   op_cost(0);
 5004   format %{ %}
 5005   interface(REG_INTER);
 5006 %}
 5007 
 5008 operand vRegD_V1()
 5009 %{
 5010   constraint(ALLOC_IN_RC(v1_reg));
 5011   match(RegD);
 5012   op_cost(0);
 5013   format %{ %}
 5014   interface(REG_INTER);
 5015 %}
 5016 
 5017 operand vRegD_V2()
 5018 %{
 5019   constraint(ALLOC_IN_RC(v2_reg));
 5020   match(RegD);
 5021   op_cost(0);
 5022   format %{ %}
 5023   interface(REG_INTER);
 5024 %}
 5025 
 5026 operand vRegD_V3()
 5027 %{
 5028   constraint(ALLOC_IN_RC(v3_reg));
 5029   match(RegD);
 5030   op_cost(0);
 5031   format %{ %}
 5032   interface(REG_INTER);
 5033 %}
 5034 
 5035 operand vRegD_V4()
 5036 %{
 5037   constraint(ALLOC_IN_RC(v4_reg));
 5038   match(RegD);
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(REG_INTER);
 5042 %}
 5043 
 5044 operand vRegD_V5()
 5045 %{
 5046   constraint(ALLOC_IN_RC(v5_reg));
 5047   match(RegD);
 5048   op_cost(0);
 5049   format %{ %}
 5050   interface(REG_INTER);
 5051 %}
 5052 
 5053 operand vRegD_V6()
 5054 %{
 5055   constraint(ALLOC_IN_RC(v6_reg));
 5056   match(RegD);
 5057   op_cost(0);
 5058   format %{ %}
 5059   interface(REG_INTER);
 5060 %}
 5061 
 5062 operand vRegD_V7()
 5063 %{
 5064   constraint(ALLOC_IN_RC(v7_reg));
 5065   match(RegD);
 5066   op_cost(0);
 5067   format %{ %}
 5068   interface(REG_INTER);
 5069 %}
 5070 
 5071 operand vRegD_V12()
 5072 %{
 5073   constraint(ALLOC_IN_RC(v12_reg));
 5074   match(RegD);
 5075   op_cost(0);
 5076   format %{ %}
 5077   interface(REG_INTER);
 5078 %}
 5079 
 5080 operand vRegD_V13()
 5081 %{
 5082   constraint(ALLOC_IN_RC(v13_reg));
 5083   match(RegD);
 5084   op_cost(0);
 5085   format %{ %}
 5086   interface(REG_INTER);
 5087 %}
 5088 
 5089 operand pReg()
 5090 %{
 5091   constraint(ALLOC_IN_RC(pr_reg));
 5092   match(RegVectMask);
 5093   match(pRegGov);
 5094   op_cost(0);
 5095   format %{ %}
 5096   interface(REG_INTER);
 5097 %}
 5098 
 5099 operand pRegGov()
 5100 %{
 5101   constraint(ALLOC_IN_RC(gov_pr));
 5102   match(RegVectMask);
 5103   match(pReg);
 5104   op_cost(0);
 5105   format %{ %}
 5106   interface(REG_INTER);
 5107 %}
 5108 
 5109 operand pRegGov_P0()
 5110 %{
 5111   constraint(ALLOC_IN_RC(p0_reg));
 5112   match(RegVectMask);
 5113   op_cost(0);
 5114   format %{ %}
 5115   interface(REG_INTER);
 5116 %}
 5117 
 5118 operand pRegGov_P1()
 5119 %{
 5120   constraint(ALLOC_IN_RC(p1_reg));
 5121   match(RegVectMask);
 5122   op_cost(0);
 5123   format %{ %}
 5124   interface(REG_INTER);
 5125 %}
 5126 
 5127 // Flags register, used as output of signed compare instructions
 5128 
 5129 // note that on AArch64 we also use this register as the output for
 5130 // for floating point compare instructions (CmpF CmpD). this ensures
 5131 // that ordered inequality tests use GT, GE, LT or LE none of which
 5132 // pass through cases where the result is unordered i.e. one or both
 5133 // inputs to the compare is a NaN. this means that the ideal code can
 5134 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5135 // (where the comparison should always fail). EQ and NE tests are
 5136 // always generated in ideal code so that unordered folds into the NE
 5137 // case, matching the behaviour of AArch64 NE.
 5138 //
 5139 // This differs from x86 where the outputs of FP compares use a
 5140 // special FP flags registers and where compares based on this
 5141 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5142 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5143 // to explicitly handle the unordered case in branches. x86 also has
 5144 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5145 
 5146 operand rFlagsReg()
 5147 %{
 5148   constraint(ALLOC_IN_RC(int_flags));
 5149   match(RegFlags);
 5150 
 5151   op_cost(0);
 5152   format %{ "RFLAGS" %}
 5153   interface(REG_INTER);
 5154 %}
 5155 
 5156 // Flags register, used as output of unsigned compare instructions
 5157 operand rFlagsRegU()
 5158 %{
 5159   constraint(ALLOC_IN_RC(int_flags));
 5160   match(RegFlags);
 5161 
 5162   op_cost(0);
 5163   format %{ "RFLAGSU" %}
 5164   interface(REG_INTER);
 5165 %}
 5166 
 5167 // Special Registers
 5168 
 5169 // Method Register
 5170 operand inline_cache_RegP(iRegP reg)
 5171 %{
 5172   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5173   match(reg);
 5174   match(iRegPNoSp);
 5175   op_cost(0);
 5176   format %{ %}
 5177   interface(REG_INTER);
 5178 %}
 5179 
 5180 // Thread Register
 5181 operand thread_RegP(iRegP reg)
 5182 %{
 5183   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5184   match(reg);
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 //----------Memory Operands----------------------------------------------------
 5191 
 5192 operand indirect(iRegP reg)
 5193 %{
 5194   constraint(ALLOC_IN_RC(ptr_reg));
 5195   match(reg);
 5196   op_cost(0);
 5197   format %{ "[$reg]" %}
 5198   interface(MEMORY_INTER) %{
 5199     base($reg);
 5200     index(0xffffffff);
 5201     scale(0x0);
 5202     disp(0x0);
 5203   %}
 5204 %}
 5205 
 5206 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5207 %{
 5208   constraint(ALLOC_IN_RC(ptr_reg));
 5209   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5210   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5211   op_cost(0);
 5212   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5213   interface(MEMORY_INTER) %{
 5214     base($reg);
 5215     index($ireg);
 5216     scale($scale);
 5217     disp(0x0);
 5218   %}
 5219 %}
 5220 
 5221 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5222 %{
 5223   constraint(ALLOC_IN_RC(ptr_reg));
 5224   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5225   match(AddP reg (LShiftL lreg scale));
 5226   op_cost(0);
 5227   format %{ "$reg, $lreg lsl($scale)" %}
 5228   interface(MEMORY_INTER) %{
 5229     base($reg);
 5230     index($lreg);
 5231     scale($scale);
 5232     disp(0x0);
 5233   %}
 5234 %}
 5235 
 5236 operand indIndexI2L(iRegP reg, iRegI ireg)
 5237 %{
 5238   constraint(ALLOC_IN_RC(ptr_reg));
 5239   match(AddP reg (ConvI2L ireg));
 5240   op_cost(0);
 5241   format %{ "$reg, $ireg, 0, I2L" %}
 5242   interface(MEMORY_INTER) %{
 5243     base($reg);
 5244     index($ireg);
 5245     scale(0x0);
 5246     disp(0x0);
 5247   %}
 5248 %}
 5249 
 5250 operand indIndex(iRegP reg, iRegL lreg)
 5251 %{
 5252   constraint(ALLOC_IN_RC(ptr_reg));
 5253   match(AddP reg lreg);
 5254   op_cost(0);
 5255   format %{ "$reg, $lreg" %}
 5256   interface(MEMORY_INTER) %{
 5257     base($reg);
 5258     index($lreg);
 5259     scale(0x0);
 5260     disp(0x0);
 5261   %}
 5262 %}
 5263 
 5264 operand indOffI1(iRegP reg, immIOffset1 off)
 5265 %{
 5266   constraint(ALLOC_IN_RC(ptr_reg));
 5267   match(AddP reg off);
 5268   op_cost(0);
 5269   format %{ "[$reg, $off]" %}
 5270   interface(MEMORY_INTER) %{
 5271     base($reg);
 5272     index(0xffffffff);
 5273     scale(0x0);
 5274     disp($off);
 5275   %}
 5276 %}
 5277 
 5278 operand indOffI2(iRegP reg, immIOffset2 off)
 5279 %{
 5280   constraint(ALLOC_IN_RC(ptr_reg));
 5281   match(AddP reg off);
 5282   op_cost(0);
 5283   format %{ "[$reg, $off]" %}
 5284   interface(MEMORY_INTER) %{
 5285     base($reg);
 5286     index(0xffffffff);
 5287     scale(0x0);
 5288     disp($off);
 5289   %}
 5290 %}
 5291 
 5292 operand indOffI4(iRegP reg, immIOffset4 off)
 5293 %{
 5294   constraint(ALLOC_IN_RC(ptr_reg));
 5295   match(AddP reg off);
 5296   op_cost(0);
 5297   format %{ "[$reg, $off]" %}
 5298   interface(MEMORY_INTER) %{
 5299     base($reg);
 5300     index(0xffffffff);
 5301     scale(0x0);
 5302     disp($off);
 5303   %}
 5304 %}
 5305 
 5306 operand indOffI8(iRegP reg, immIOffset8 off)
 5307 %{
 5308   constraint(ALLOC_IN_RC(ptr_reg));
 5309   match(AddP reg off);
 5310   op_cost(0);
 5311   format %{ "[$reg, $off]" %}
 5312   interface(MEMORY_INTER) %{
 5313     base($reg);
 5314     index(0xffffffff);
 5315     scale(0x0);
 5316     disp($off);
 5317   %}
 5318 %}
 5319 
 5320 operand indOffI16(iRegP reg, immIOffset16 off)
 5321 %{
 5322   constraint(ALLOC_IN_RC(ptr_reg));
 5323   match(AddP reg off);
 5324   op_cost(0);
 5325   format %{ "[$reg, $off]" %}
 5326   interface(MEMORY_INTER) %{
 5327     base($reg);
 5328     index(0xffffffff);
 5329     scale(0x0);
 5330     disp($off);
 5331   %}
 5332 %}
 5333 
 5334 operand indOffL1(iRegP reg, immLoffset1 off)
 5335 %{
 5336   constraint(ALLOC_IN_RC(ptr_reg));
 5337   match(AddP reg off);
 5338   op_cost(0);
 5339   format %{ "[$reg, $off]" %}
 5340   interface(MEMORY_INTER) %{
 5341     base($reg);
 5342     index(0xffffffff);
 5343     scale(0x0);
 5344     disp($off);
 5345   %}
 5346 %}
 5347 
 5348 operand indOffL2(iRegP reg, immLoffset2 off)
 5349 %{
 5350   constraint(ALLOC_IN_RC(ptr_reg));
 5351   match(AddP reg off);
 5352   op_cost(0);
 5353   format %{ "[$reg, $off]" %}
 5354   interface(MEMORY_INTER) %{
 5355     base($reg);
 5356     index(0xffffffff);
 5357     scale(0x0);
 5358     disp($off);
 5359   %}
 5360 %}
 5361 
 5362 operand indOffL4(iRegP reg, immLoffset4 off)
 5363 %{
 5364   constraint(ALLOC_IN_RC(ptr_reg));
 5365   match(AddP reg off);
 5366   op_cost(0);
 5367   format %{ "[$reg, $off]" %}
 5368   interface(MEMORY_INTER) %{
 5369     base($reg);
 5370     index(0xffffffff);
 5371     scale(0x0);
 5372     disp($off);
 5373   %}
 5374 %}
 5375 
 5376 operand indOffL8(iRegP reg, immLoffset8 off)
 5377 %{
 5378   constraint(ALLOC_IN_RC(ptr_reg));
 5379   match(AddP reg off);
 5380   op_cost(0);
 5381   format %{ "[$reg, $off]" %}
 5382   interface(MEMORY_INTER) %{
 5383     base($reg);
 5384     index(0xffffffff);
 5385     scale(0x0);
 5386     disp($off);
 5387   %}
 5388 %}
 5389 
 5390 operand indOffL16(iRegP reg, immLoffset16 off)
 5391 %{
 5392   constraint(ALLOC_IN_RC(ptr_reg));
 5393   match(AddP reg off);
 5394   op_cost(0);
 5395   format %{ "[$reg, $off]" %}
 5396   interface(MEMORY_INTER) %{
 5397     base($reg);
 5398     index(0xffffffff);
 5399     scale(0x0);
 5400     disp($off);
 5401   %}
 5402 %}
 5403 
 5404 operand indirectX2P(iRegL reg)
 5405 %{
 5406   constraint(ALLOC_IN_RC(ptr_reg));
 5407   match(CastX2P reg);
 5408   op_cost(0);
 5409   format %{ "[$reg]\t# long -> ptr" %}
 5410   interface(MEMORY_INTER) %{
 5411     base($reg);
 5412     index(0xffffffff);
 5413     scale(0x0);
 5414     disp(0x0);
 5415   %}
 5416 %}
 5417 
 5418 operand indOffX2P(iRegL reg, immLOffset off)
 5419 %{
 5420   constraint(ALLOC_IN_RC(ptr_reg));
 5421   match(AddP (CastX2P reg) off);
 5422   op_cost(0);
 5423   format %{ "[$reg, $off]\t# long -> ptr" %}
 5424   interface(MEMORY_INTER) %{
 5425     base($reg);
 5426     index(0xffffffff);
 5427     scale(0x0);
 5428     disp($off);
 5429   %}
 5430 %}
 5431 
 5432 operand indirectN(iRegN reg)
 5433 %{
 5434   predicate(CompressedOops::shift() == 0);
 5435   constraint(ALLOC_IN_RC(ptr_reg));
 5436   match(DecodeN reg);
 5437   op_cost(0);
 5438   format %{ "[$reg]\t# narrow" %}
 5439   interface(MEMORY_INTER) %{
 5440     base($reg);
 5441     index(0xffffffff);
 5442     scale(0x0);
 5443     disp(0x0);
 5444   %}
 5445 %}
 5446 
 5447 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5448 %{
 5449   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5450   constraint(ALLOC_IN_RC(ptr_reg));
 5451   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5452   op_cost(0);
 5453   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5454   interface(MEMORY_INTER) %{
 5455     base($reg);
 5456     index($ireg);
 5457     scale($scale);
 5458     disp(0x0);
 5459   %}
 5460 %}
 5461 
 5462 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5463 %{
 5464   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5465   constraint(ALLOC_IN_RC(ptr_reg));
 5466   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5467   op_cost(0);
 5468   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5469   interface(MEMORY_INTER) %{
 5470     base($reg);
 5471     index($lreg);
 5472     scale($scale);
 5473     disp(0x0);
 5474   %}
 5475 %}
 5476 
 5477 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5478 %{
 5479   predicate(CompressedOops::shift() == 0);
 5480   constraint(ALLOC_IN_RC(ptr_reg));
 5481   match(AddP (DecodeN reg) (ConvI2L ireg));
 5482   op_cost(0);
 5483   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5484   interface(MEMORY_INTER) %{
 5485     base($reg);
 5486     index($ireg);
 5487     scale(0x0);
 5488     disp(0x0);
 5489   %}
 5490 %}
 5491 
 5492 operand indIndexN(iRegN reg, iRegL lreg)
 5493 %{
 5494   predicate(CompressedOops::shift() == 0);
 5495   constraint(ALLOC_IN_RC(ptr_reg));
 5496   match(AddP (DecodeN reg) lreg);
 5497   op_cost(0);
 5498   format %{ "$reg, $lreg\t# narrow" %}
 5499   interface(MEMORY_INTER) %{
 5500     base($reg);
 5501     index($lreg);
 5502     scale(0x0);
 5503     disp(0x0);
 5504   %}
 5505 %}
 5506 
 5507 operand indOffIN(iRegN reg, immIOffset off)
 5508 %{
 5509   predicate(CompressedOops::shift() == 0);
 5510   constraint(ALLOC_IN_RC(ptr_reg));
 5511   match(AddP (DecodeN reg) off);
 5512   op_cost(0);
 5513   format %{ "[$reg, $off]\t# narrow" %}
 5514   interface(MEMORY_INTER) %{
 5515     base($reg);
 5516     index(0xffffffff);
 5517     scale(0x0);
 5518     disp($off);
 5519   %}
 5520 %}
 5521 
 5522 operand indOffLN(iRegN reg, immLOffset off)
 5523 %{
 5524   predicate(CompressedOops::shift() == 0);
 5525   constraint(ALLOC_IN_RC(ptr_reg));
 5526   match(AddP (DecodeN reg) off);
 5527   op_cost(0);
 5528   format %{ "[$reg, $off]\t# narrow" %}
 5529   interface(MEMORY_INTER) %{
 5530     base($reg);
 5531     index(0xffffffff);
 5532     scale(0x0);
 5533     disp($off);
 5534   %}
 5535 %}
 5536 
 5537 
 5538 //----------Special Memory Operands--------------------------------------------
 5539 // Stack Slot Operand - This operand is used for loading and storing temporary
 5540 //                      values on the stack where a match requires a value to
 5541 //                      flow through memory.
 5542 operand stackSlotP(sRegP reg)
 5543 %{
 5544   constraint(ALLOC_IN_RC(stack_slots));
 5545   op_cost(100);
 5546   // No match rule because this operand is only generated in matching
 5547   // match(RegP);
 5548   format %{ "[$reg]" %}
 5549   interface(MEMORY_INTER) %{
 5550     base(0x1e);  // RSP
 5551     index(0x0);  // No Index
 5552     scale(0x0);  // No Scale
 5553     disp($reg);  // Stack Offset
 5554   %}
 5555 %}
 5556 
 5557 operand stackSlotI(sRegI reg)
 5558 %{
 5559   constraint(ALLOC_IN_RC(stack_slots));
 5560   // No match rule because this operand is only generated in matching
 5561   // match(RegI);
 5562   format %{ "[$reg]" %}
 5563   interface(MEMORY_INTER) %{
 5564     base(0x1e);  // RSP
 5565     index(0x0);  // No Index
 5566     scale(0x0);  // No Scale
 5567     disp($reg);  // Stack Offset
 5568   %}
 5569 %}
 5570 
 5571 operand stackSlotF(sRegF reg)
 5572 %{
 5573   constraint(ALLOC_IN_RC(stack_slots));
 5574   // No match rule because this operand is only generated in matching
 5575   // match(RegF);
 5576   format %{ "[$reg]" %}
 5577   interface(MEMORY_INTER) %{
 5578     base(0x1e);  // RSP
 5579     index(0x0);  // No Index
 5580     scale(0x0);  // No Scale
 5581     disp($reg);  // Stack Offset
 5582   %}
 5583 %}
 5584 
 5585 operand stackSlotD(sRegD reg)
 5586 %{
 5587   constraint(ALLOC_IN_RC(stack_slots));
 5588   // No match rule because this operand is only generated in matching
 5589   // match(RegD);
 5590   format %{ "[$reg]" %}
 5591   interface(MEMORY_INTER) %{
 5592     base(0x1e);  // RSP
 5593     index(0x0);  // No Index
 5594     scale(0x0);  // No Scale
 5595     disp($reg);  // Stack Offset
 5596   %}
 5597 %}
 5598 
 5599 operand stackSlotL(sRegL reg)
 5600 %{
 5601   constraint(ALLOC_IN_RC(stack_slots));
 5602   // No match rule because this operand is only generated in matching
 5603   // match(RegL);
 5604   format %{ "[$reg]" %}
 5605   interface(MEMORY_INTER) %{
 5606     base(0x1e);  // RSP
 5607     index(0x0);  // No Index
 5608     scale(0x0);  // No Scale
 5609     disp($reg);  // Stack Offset
 5610   %}
 5611 %}
 5612 
 5613 // Operands for expressing Control Flow
 5614 // NOTE: Label is a predefined operand which should not be redefined in
 5615 //       the AD file. It is generically handled within the ADLC.
 5616 
 5617 //----------Conditional Branch Operands----------------------------------------
 5618 // Comparison Op  - This is the operation of the comparison, and is limited to
 5619 //                  the following set of codes:
 5620 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5621 //
 5622 // Other attributes of the comparison, such as unsignedness, are specified
 5623 // by the comparison instruction that sets a condition code flags register.
 5624 // That result is represented by a flags operand whose subtype is appropriate
 5625 // to the unsignedness (etc.) of the comparison.
 5626 //
 5627 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5628 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5629 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5630 
 5631 // used for signed integral comparisons and fp comparisons
 5632 
 5633 operand cmpOp()
 5634 %{
 5635   match(Bool);
 5636 
 5637   format %{ "" %}
 5638   interface(COND_INTER) %{
 5639     equal(0x0, "eq");
 5640     not_equal(0x1, "ne");
 5641     less(0xb, "lt");
 5642     greater_equal(0xa, "ge");
 5643     less_equal(0xd, "le");
 5644     greater(0xc, "gt");
 5645     overflow(0x6, "vs");
 5646     no_overflow(0x7, "vc");
 5647   %}
 5648 %}
 5649 
 5650 // used for unsigned integral comparisons
 5651 
 5652 operand cmpOpU()
 5653 %{
 5654   match(Bool);
 5655 
 5656   format %{ "" %}
 5657   interface(COND_INTER) %{
 5658     equal(0x0, "eq");
 5659     not_equal(0x1, "ne");
 5660     less(0x3, "lo");
 5661     greater_equal(0x2, "hs");
 5662     less_equal(0x9, "ls");
 5663     greater(0x8, "hi");
 5664     overflow(0x6, "vs");
 5665     no_overflow(0x7, "vc");
 5666   %}
 5667 %}
 5668 
 5669 // used for certain integral comparisons which can be
 5670 // converted to cbxx or tbxx instructions
 5671 
 5672 operand cmpOpEqNe()
 5673 %{
 5674   match(Bool);
 5675   op_cost(0);
 5676   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5677             || n->as_Bool()->_test._test == BoolTest::eq);
 5678 
 5679   format %{ "" %}
 5680   interface(COND_INTER) %{
 5681     equal(0x0, "eq");
 5682     not_equal(0x1, "ne");
 5683     less(0xb, "lt");
 5684     greater_equal(0xa, "ge");
 5685     less_equal(0xd, "le");
 5686     greater(0xc, "gt");
 5687     overflow(0x6, "vs");
 5688     no_overflow(0x7, "vc");
 5689   %}
 5690 %}
 5691 
 5692 // used for certain integral comparisons which can be
 5693 // converted to cbxx or tbxx instructions
 5694 
 5695 operand cmpOpLtGe()
 5696 %{
 5697   match(Bool);
 5698   op_cost(0);
 5699 
 5700   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5701             || n->as_Bool()->_test._test == BoolTest::ge);
 5702 
 5703   format %{ "" %}
 5704   interface(COND_INTER) %{
 5705     equal(0x0, "eq");
 5706     not_equal(0x1, "ne");
 5707     less(0xb, "lt");
 5708     greater_equal(0xa, "ge");
 5709     less_equal(0xd, "le");
 5710     greater(0xc, "gt");
 5711     overflow(0x6, "vs");
 5712     no_overflow(0x7, "vc");
 5713   %}
 5714 %}
 5715 
 5716 // used for certain unsigned integral comparisons which can be
 5717 // converted to cbxx or tbxx instructions
 5718 
 5719 operand cmpOpUEqNeLeGt()
 5720 %{
 5721   match(Bool);
 5722   op_cost(0);
 5723 
 5724   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5725             n->as_Bool()->_test._test == BoolTest::ne ||
 5726             n->as_Bool()->_test._test == BoolTest::le ||
 5727             n->as_Bool()->_test._test == BoolTest::gt);
 5728 
 5729   format %{ "" %}
 5730   interface(COND_INTER) %{
 5731     equal(0x0, "eq");
 5732     not_equal(0x1, "ne");
 5733     less(0x3, "lo");
 5734     greater_equal(0x2, "hs");
 5735     less_equal(0x9, "ls");
 5736     greater(0x8, "hi");
 5737     overflow(0x6, "vs");
 5738     no_overflow(0x7, "vc");
 5739   %}
 5740 %}
 5741 
 5742 // Special operand allowing long args to int ops to be truncated for free
 5743 
 5744 operand iRegL2I(iRegL reg) %{
 5745 
 5746   op_cost(0);
 5747 
 5748   match(ConvL2I reg);
 5749 
 5750   format %{ "l2i($reg)" %}
 5751 
 5752   interface(REG_INTER)
 5753 %}
 5754 
 5755 operand iRegL2P(iRegL reg) %{
 5756 
 5757   op_cost(0);
 5758 
 5759   match(CastX2P reg);
 5760 
 5761   format %{ "l2p($reg)" %}
 5762 
 5763   interface(REG_INTER)
 5764 %}
 5765 
 5766 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5767 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5768 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5769 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5770 
 5771 //----------OPERAND CLASSES----------------------------------------------------
 5772 // Operand Classes are groups of operands that are used as to simplify
 5773 // instruction definitions by not requiring the AD writer to specify
 5774 // separate instructions for every form of operand when the
 5775 // instruction accepts multiple operand types with the same basic
 5776 // encoding and format. The classic case of this is memory operands.
 5777 
 5778 // memory is used to define read/write location for load/store
 5779 // instruction defs. we can turn a memory op into an Address
 5780 
 5781 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5782                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5783 
 5784 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5785                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5786 
 5787 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5788                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5789 
 5790 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5791                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5792 
 5793 // All of the memory operands. For the pipeline description.
 5794 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5795                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5796                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5797 
 5798 
 5799 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5800 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5801 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5802 // can be elided because the 32-bit instruction will just employ the
 5803 // lower 32 bits anyway.
 5804 //
 5805 // n.b. this does not elide all L2I conversions. if the truncated
 5806 // value is consumed by more than one operation then the ConvL2I
 5807 // cannot be bundled into the consuming nodes so an l2i gets planted
 5808 // (actually a movw $dst $src) and the downstream instructions consume
 5809 // the result of the l2i as an iRegI input. That's a shame since the
 5810 // movw is actually redundant but its not too costly.
 5811 
 5812 opclass iRegIorL2I(iRegI, iRegL2I);
 5813 opclass iRegPorL2P(iRegP, iRegL2P);
 5814 
 5815 //----------PIPELINE-----------------------------------------------------------
 5816 // Rules which define the behavior of the target architectures pipeline.
 5817 
 5818 // For specific pipelines, eg A53, define the stages of that pipeline
 5819 //pipe_desc(ISS, EX1, EX2, WR);
 5820 #define ISS S0
 5821 #define EX1 S1
 5822 #define EX2 S2
 5823 #define WR  S3
 5824 
 5825 // Integer ALU reg operation
 5826 pipeline %{
 5827 
 5828 attributes %{
 5829   // ARM instructions are of fixed length
 5830   fixed_size_instructions;        // Fixed size instructions TODO does
 5831   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5832   // ARM instructions come in 32-bit word units
 5833   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5834   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5835   instruction_fetch_units = 1;       // of 64 bytes
 5836 
 5837   // List of nop instructions
 5838   nops( MachNop );
 5839 %}
 5840 
 5841 // We don't use an actual pipeline model so don't care about resources
 5842 // or description. we do use pipeline classes to introduce fixed
 5843 // latencies
 5844 
 5845 //----------RESOURCES----------------------------------------------------------
 5846 // Resources are the functional units available to the machine
 5847 
 5848 resources( INS0, INS1, INS01 = INS0 | INS1,
 5849            ALU0, ALU1, ALU = ALU0 | ALU1,
 5850            MAC,
 5851            DIV,
 5852            BRANCH,
 5853            LDST,
 5854            NEON_FP);
 5855 
 5856 //----------PIPELINE DESCRIPTION-----------------------------------------------
 5857 // Pipeline Description specifies the stages in the machine's pipeline
 5858 
 5859 // Define the pipeline as a generic 6 stage pipeline
 5860 pipe_desc(S0, S1, S2, S3, S4, S5);
 5861 
 5862 //----------PIPELINE CLASSES---------------------------------------------------
 5863 // Pipeline Classes describe the stages in which input and output are
 5864 // referenced by the hardware pipeline.
 5865 
 5866 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 5867 %{
 5868   single_instruction;
 5869   src1   : S1(read);
 5870   src2   : S2(read);
 5871   dst    : S5(write);
 5872   INS01  : ISS;
 5873   NEON_FP : S5;
 5874 %}
 5875 
 5876 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 5877 %{
 5878   single_instruction;
 5879   src1   : S1(read);
 5880   src2   : S2(read);
 5881   dst    : S5(write);
 5882   INS01  : ISS;
 5883   NEON_FP : S5;
 5884 %}
 5885 
 5886 pipe_class fp_uop_s(vRegF dst, vRegF src)
 5887 %{
 5888   single_instruction;
 5889   src    : S1(read);
 5890   dst    : S5(write);
 5891   INS01  : ISS;
 5892   NEON_FP : S5;
 5893 %}
 5894 
 5895 pipe_class fp_uop_d(vRegD dst, vRegD src)
 5896 %{
 5897   single_instruction;
 5898   src    : S1(read);
 5899   dst    : S5(write);
 5900   INS01  : ISS;
 5901   NEON_FP : S5;
 5902 %}
 5903 
 5904 pipe_class fp_d2f(vRegF dst, vRegD src)
 5905 %{
 5906   single_instruction;
 5907   src    : S1(read);
 5908   dst    : S5(write);
 5909   INS01  : ISS;
 5910   NEON_FP : S5;
 5911 %}
 5912 
 5913 pipe_class fp_f2d(vRegD dst, vRegF src)
 5914 %{
 5915   single_instruction;
 5916   src    : S1(read);
 5917   dst    : S5(write);
 5918   INS01  : ISS;
 5919   NEON_FP : S5;
 5920 %}
 5921 
 5922 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 5923 %{
 5924   single_instruction;
 5925   src    : S1(read);
 5926   dst    : S5(write);
 5927   INS01  : ISS;
 5928   NEON_FP : S5;
 5929 %}
 5930 
 5931 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 5932 %{
 5933   single_instruction;
 5934   src    : S1(read);
 5935   dst    : S5(write);
 5936   INS01  : ISS;
 5937   NEON_FP : S5;
 5938 %}
 5939 
 5940 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 5941 %{
 5942   single_instruction;
 5943   src    : S1(read);
 5944   dst    : S5(write);
 5945   INS01  : ISS;
 5946   NEON_FP : S5;
 5947 %}
 5948 
 5949 pipe_class fp_l2f(vRegF dst, iRegL src)
 5950 %{
 5951   single_instruction;
 5952   src    : S1(read);
 5953   dst    : S5(write);
 5954   INS01  : ISS;
 5955   NEON_FP : S5;
 5956 %}
 5957 
 5958 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 5959 %{
 5960   single_instruction;
 5961   src    : S1(read);
 5962   dst    : S5(write);
 5963   INS01  : ISS;
 5964   NEON_FP : S5;
 5965 %}
 5966 
 5967 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 5968 %{
 5969   single_instruction;
 5970   src    : S1(read);
 5971   dst    : S5(write);
 5972   INS01  : ISS;
 5973   NEON_FP : S5;
 5974 %}
 5975 
 5976 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 5977 %{
 5978   single_instruction;
 5979   src    : S1(read);
 5980   dst    : S5(write);
 5981   INS01  : ISS;
 5982   NEON_FP : S5;
 5983 %}
 5984 
 5985 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 5986 %{
 5987   single_instruction;
 5988   src    : S1(read);
 5989   dst    : S5(write);
 5990   INS01  : ISS;
 5991   NEON_FP : S5;
 5992 %}
 5993 
 5994 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 5995 %{
 5996   single_instruction;
 5997   src1   : S1(read);
 5998   src2   : S2(read);
 5999   dst    : S5(write);
 6000   INS0   : ISS;
 6001   NEON_FP : S5;
 6002 %}
 6003 
 6004 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6005 %{
 6006   single_instruction;
 6007   src1   : S1(read);
 6008   src2   : S2(read);
 6009   dst    : S5(write);
 6010   INS0   : ISS;
 6011   NEON_FP : S5;
 6012 %}
 6013 
 6014 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6015 %{
 6016   single_instruction;
 6017   cr     : S1(read);
 6018   src1   : S1(read);
 6019   src2   : S1(read);
 6020   dst    : S3(write);
 6021   INS01  : ISS;
 6022   NEON_FP : S3;
 6023 %}
 6024 
 6025 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6026 %{
 6027   single_instruction;
 6028   cr     : S1(read);
 6029   src1   : S1(read);
 6030   src2   : S1(read);
 6031   dst    : S3(write);
 6032   INS01  : ISS;
 6033   NEON_FP : S3;
 6034 %}
 6035 
 6036 pipe_class fp_imm_s(vRegF dst)
 6037 %{
 6038   single_instruction;
 6039   dst    : S3(write);
 6040   INS01  : ISS;
 6041   NEON_FP : S3;
 6042 %}
 6043 
 6044 pipe_class fp_imm_d(vRegD dst)
 6045 %{
 6046   single_instruction;
 6047   dst    : S3(write);
 6048   INS01  : ISS;
 6049   NEON_FP : S3;
 6050 %}
 6051 
 6052 pipe_class fp_load_constant_s(vRegF dst)
 6053 %{
 6054   single_instruction;
 6055   dst    : S4(write);
 6056   INS01  : ISS;
 6057   NEON_FP : S4;
 6058 %}
 6059 
 6060 pipe_class fp_load_constant_d(vRegD dst)
 6061 %{
 6062   single_instruction;
 6063   dst    : S4(write);
 6064   INS01  : ISS;
 6065   NEON_FP : S4;
 6066 %}
 6067 
 6068 //------- Integer ALU operations --------------------------
 6069 
 6070 // Integer ALU reg-reg operation
 6071 // Operands needed in EX1, result generated in EX2
 6072 // Eg.  ADD     x0, x1, x2
 6073 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6074 %{
 6075   single_instruction;
 6076   dst    : EX2(write);
 6077   src1   : EX1(read);
 6078   src2   : EX1(read);
 6079   INS01  : ISS; // Dual issue as instruction 0 or 1
 6080   ALU    : EX2;
 6081 %}
 6082 
 6083 // Integer ALU reg-reg operation with constant shift
 6084 // Shifted register must be available in LATE_ISS instead of EX1
 6085 // Eg.  ADD     x0, x1, x2, LSL #2
 6086 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6087 %{
 6088   single_instruction;
 6089   dst    : EX2(write);
 6090   src1   : EX1(read);
 6091   src2   : ISS(read);
 6092   INS01  : ISS;
 6093   ALU    : EX2;
 6094 %}
 6095 
 6096 // Integer ALU reg operation with constant shift
 6097 // Eg.  LSL     x0, x1, #shift
 6098 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6099 %{
 6100   single_instruction;
 6101   dst    : EX2(write);
 6102   src1   : ISS(read);
 6103   INS01  : ISS;
 6104   ALU    : EX2;
 6105 %}
 6106 
 6107 // Integer ALU reg-reg operation with variable shift
 6108 // Both operands must be available in LATE_ISS instead of EX1
 6109 // Result is available in EX1 instead of EX2
 6110 // Eg.  LSLV    x0, x1, x2
 6111 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6112 %{
 6113   single_instruction;
 6114   dst    : EX1(write);
 6115   src1   : ISS(read);
 6116   src2   : ISS(read);
 6117   INS01  : ISS;
 6118   ALU    : EX1;
 6119 %}
 6120 
 6121 // Integer ALU reg-reg operation with extract
 6122 // As for _vshift above, but result generated in EX2
 6123 // Eg.  EXTR    x0, x1, x2, #N
 6124 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6125 %{
 6126   single_instruction;
 6127   dst    : EX2(write);
 6128   src1   : ISS(read);
 6129   src2   : ISS(read);
 6130   INS1   : ISS; // Can only dual issue as Instruction 1
 6131   ALU    : EX1;
 6132 %}
 6133 
 6134 // Integer ALU reg operation
 6135 // Eg.  NEG     x0, x1
 6136 pipe_class ialu_reg(iRegI dst, iRegI src)
 6137 %{
 6138   single_instruction;
 6139   dst    : EX2(write);
 6140   src    : EX1(read);
 6141   INS01  : ISS;
 6142   ALU    : EX2;
 6143 %}
 6144 
 6145 // Integer ALU reg mmediate operation
 6146 // Eg.  ADD     x0, x1, #N
 6147 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6148 %{
 6149   single_instruction;
 6150   dst    : EX2(write);
 6151   src1   : EX1(read);
 6152   INS01  : ISS;
 6153   ALU    : EX2;
 6154 %}
 6155 
 6156 // Integer ALU immediate operation (no source operands)
 6157 // Eg.  MOV     x0, #N
 6158 pipe_class ialu_imm(iRegI dst)
 6159 %{
 6160   single_instruction;
 6161   dst    : EX1(write);
 6162   INS01  : ISS;
 6163   ALU    : EX1;
 6164 %}
 6165 
 6166 //------- Compare operation -------------------------------
 6167 
 6168 // Compare reg-reg
 6169 // Eg.  CMP     x0, x1
 6170 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6171 %{
 6172   single_instruction;
 6173 //  fixed_latency(16);
 6174   cr     : EX2(write);
 6175   op1    : EX1(read);
 6176   op2    : EX1(read);
 6177   INS01  : ISS;
 6178   ALU    : EX2;
 6179 %}
 6180 
 6181 // Compare reg-reg
 6182 // Eg.  CMP     x0, #N
 6183 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6184 %{
 6185   single_instruction;
 6186 //  fixed_latency(16);
 6187   cr     : EX2(write);
 6188   op1    : EX1(read);
 6189   INS01  : ISS;
 6190   ALU    : EX2;
 6191 %}
 6192 
 6193 //------- Conditional instructions ------------------------
 6194 
 6195 // Conditional no operands
 6196 // Eg.  CSINC   x0, zr, zr, <cond>
 6197 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6198 %{
 6199   single_instruction;
 6200   cr     : EX1(read);
 6201   dst    : EX2(write);
 6202   INS01  : ISS;
 6203   ALU    : EX2;
 6204 %}
 6205 
 6206 // Conditional 2 operand
 6207 // EG.  CSEL    X0, X1, X2, <cond>
 6208 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6209 %{
 6210   single_instruction;
 6211   cr     : EX1(read);
 6212   src1   : EX1(read);
 6213   src2   : EX1(read);
 6214   dst    : EX2(write);
 6215   INS01  : ISS;
 6216   ALU    : EX2;
 6217 %}
 6218 
 6219 // Conditional 2 operand
 6220 // EG.  CSEL    X0, X1, X2, <cond>
 6221 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6222 %{
 6223   single_instruction;
 6224   cr     : EX1(read);
 6225   src    : EX1(read);
 6226   dst    : EX2(write);
 6227   INS01  : ISS;
 6228   ALU    : EX2;
 6229 %}
 6230 
 6231 //------- Multiply pipeline operations --------------------
 6232 
 6233 // Multiply reg-reg
 6234 // Eg.  MUL     w0, w1, w2
 6235 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6236 %{
 6237   single_instruction;
 6238   dst    : WR(write);
 6239   src1   : ISS(read);
 6240   src2   : ISS(read);
 6241   INS01  : ISS;
 6242   MAC    : WR;
 6243 %}
 6244 
 6245 // Multiply accumulate
 6246 // Eg.  MADD    w0, w1, w2, w3
 6247 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6248 %{
 6249   single_instruction;
 6250   dst    : WR(write);
 6251   src1   : ISS(read);
 6252   src2   : ISS(read);
 6253   src3   : ISS(read);
 6254   INS01  : ISS;
 6255   MAC    : WR;
 6256 %}
 6257 
 6258 // Eg.  MUL     w0, w1, w2
 6259 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6260 %{
 6261   single_instruction;
 6262   fixed_latency(3); // Maximum latency for 64 bit mul
 6263   dst    : WR(write);
 6264   src1   : ISS(read);
 6265   src2   : ISS(read);
 6266   INS01  : ISS;
 6267   MAC    : WR;
 6268 %}
 6269 
 6270 // Multiply accumulate
 6271 // Eg.  MADD    w0, w1, w2, w3
 6272 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6273 %{
 6274   single_instruction;
 6275   fixed_latency(3); // Maximum latency for 64 bit mul
 6276   dst    : WR(write);
 6277   src1   : ISS(read);
 6278   src2   : ISS(read);
 6279   src3   : ISS(read);
 6280   INS01  : ISS;
 6281   MAC    : WR;
 6282 %}
 6283 
 6284 //------- Divide pipeline operations --------------------
 6285 
 6286 // Eg.  SDIV    w0, w1, w2
 6287 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6288 %{
 6289   single_instruction;
 6290   fixed_latency(8); // Maximum latency for 32 bit divide
 6291   dst    : WR(write);
 6292   src1   : ISS(read);
 6293   src2   : ISS(read);
 6294   INS0   : ISS; // Can only dual issue as instruction 0
 6295   DIV    : WR;
 6296 %}
 6297 
 6298 // Eg.  SDIV    x0, x1, x2
 6299 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6300 %{
 6301   single_instruction;
 6302   fixed_latency(16); // Maximum latency for 64 bit divide
 6303   dst    : WR(write);
 6304   src1   : ISS(read);
 6305   src2   : ISS(read);
 6306   INS0   : ISS; // Can only dual issue as instruction 0
 6307   DIV    : WR;
 6308 %}
 6309 
 6310 //------- Load pipeline operations ------------------------
 6311 
 6312 // Load - prefetch
 6313 // Eg.  PFRM    <mem>
 6314 pipe_class iload_prefetch(memory mem)
 6315 %{
 6316   single_instruction;
 6317   mem    : ISS(read);
 6318   INS01  : ISS;
 6319   LDST   : WR;
 6320 %}
 6321 
 6322 // Load - reg, mem
 6323 // Eg.  LDR     x0, <mem>
 6324 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6325 %{
 6326   single_instruction;
 6327   dst    : WR(write);
 6328   mem    : ISS(read);
 6329   INS01  : ISS;
 6330   LDST   : WR;
 6331 %}
 6332 
 6333 // Load - reg, reg
 6334 // Eg.  LDR     x0, [sp, x1]
 6335 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6336 %{
 6337   single_instruction;
 6338   dst    : WR(write);
 6339   src    : ISS(read);
 6340   INS01  : ISS;
 6341   LDST   : WR;
 6342 %}
 6343 
 6344 //------- Store pipeline operations -----------------------
 6345 
 6346 // Store - zr, mem
 6347 // Eg.  STR     zr, <mem>
 6348 pipe_class istore_mem(memory mem)
 6349 %{
 6350   single_instruction;
 6351   mem    : ISS(read);
 6352   INS01  : ISS;
 6353   LDST   : WR;
 6354 %}
 6355 
 6356 // Store - reg, mem
 6357 // Eg.  STR     x0, <mem>
 6358 pipe_class istore_reg_mem(iRegI src, memory mem)
 6359 %{
 6360   single_instruction;
 6361   mem    : ISS(read);
 6362   src    : EX2(read);
 6363   INS01  : ISS;
 6364   LDST   : WR;
 6365 %}
 6366 
 6367 // Store - reg, reg
 6368 // Eg. STR      x0, [sp, x1]
 6369 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6370 %{
 6371   single_instruction;
 6372   dst    : ISS(read);
 6373   src    : EX2(read);
 6374   INS01  : ISS;
 6375   LDST   : WR;
 6376 %}
 6377 
 6378 //------- Store pipeline operations -----------------------
 6379 
 6380 // Branch
 6381 pipe_class pipe_branch()
 6382 %{
 6383   single_instruction;
 6384   INS01  : ISS;
 6385   BRANCH : EX1;
 6386 %}
 6387 
 6388 // Conditional branch
 6389 pipe_class pipe_branch_cond(rFlagsReg cr)
 6390 %{
 6391   single_instruction;
 6392   cr     : EX1(read);
 6393   INS01  : ISS;
 6394   BRANCH : EX1;
 6395 %}
 6396 
 6397 // Compare & Branch
 6398 // EG.  CBZ/CBNZ
 6399 pipe_class pipe_cmp_branch(iRegI op1)
 6400 %{
 6401   single_instruction;
 6402   op1    : EX1(read);
 6403   INS01  : ISS;
 6404   BRANCH : EX1;
 6405 %}
 6406 
 6407 //------- Synchronisation operations ----------------------
 6408 
 6409 // Any operation requiring serialization.
 6410 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6411 pipe_class pipe_serial()
 6412 %{
 6413   single_instruction;
 6414   force_serialization;
 6415   fixed_latency(16);
 6416   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6417   LDST   : WR;
 6418 %}
 6419 
 6420 // Generic big/slow expanded idiom - also serialized
 6421 pipe_class pipe_slow()
 6422 %{
 6423   instruction_count(10);
 6424   multiple_bundles;
 6425   force_serialization;
 6426   fixed_latency(16);
 6427   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6428   LDST   : WR;
 6429 %}
 6430 
 6431 // Empty pipeline class
 6432 pipe_class pipe_class_empty()
 6433 %{
 6434   single_instruction;
 6435   fixed_latency(0);
 6436 %}
 6437 
 6438 // Default pipeline class.
 6439 pipe_class pipe_class_default()
 6440 %{
 6441   single_instruction;
 6442   fixed_latency(2);
 6443 %}
 6444 
 6445 // Pipeline class for compares.
 6446 pipe_class pipe_class_compare()
 6447 %{
 6448   single_instruction;
 6449   fixed_latency(16);
 6450 %}
 6451 
 6452 // Pipeline class for memory operations.
 6453 pipe_class pipe_class_memory()
 6454 %{
 6455   single_instruction;
 6456   fixed_latency(16);
 6457 %}
 6458 
 6459 // Pipeline class for call.
 6460 pipe_class pipe_class_call()
 6461 %{
 6462   single_instruction;
 6463   fixed_latency(100);
 6464 %}
 6465 
 6466 // Define the class for the Nop node.
 6467 define %{
 6468    MachNop = pipe_class_empty;
 6469 %}
 6470 
 6471 %}
 6472 //----------INSTRUCTIONS-------------------------------------------------------
 6473 //
 6474 // match      -- States which machine-independent subtree may be replaced
 6475 //               by this instruction.
 6476 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6477 //               selection to identify a minimum cost tree of machine
 6478 //               instructions that matches a tree of machine-independent
 6479 //               instructions.
 6480 // format     -- A string providing the disassembly for this instruction.
 6481 //               The value of an instruction's operand may be inserted
 6482 //               by referring to it with a '$' prefix.
 6483 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6484 //               to within an encode class as $primary, $secondary, and $tertiary
 6485 //               rrspectively.  The primary opcode is commonly used to
 6486 //               indicate the type of machine instruction, while secondary
 6487 //               and tertiary are often used for prefix options or addressing
 6488 //               modes.
 6489 // ins_encode -- A list of encode classes with parameters. The encode class
 6490 //               name must have been defined in an 'enc_class' specification
 6491 //               in the encode section of the architecture description.
 6492 
 6493 // ============================================================================
 6494 // Memory (Load/Store) Instructions
 6495 
 6496 // Load Instructions
 6497 
 6498 // Load Byte (8 bit signed)
 6499 instruct loadB(iRegINoSp dst, memory1 mem)
 6500 %{
 6501   match(Set dst (LoadB mem));
 6502   predicate(!needs_acquiring_load(n));
 6503 
 6504   ins_cost(4 * INSN_COST);
 6505   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6506 
 6507   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6508 
 6509   ins_pipe(iload_reg_mem);
 6510 %}
 6511 
 6512 // Load Byte (8 bit signed) into long
 6513 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6514 %{
 6515   match(Set dst (ConvI2L (LoadB mem)));
 6516   predicate(!needs_acquiring_load(n->in(1)));
 6517 
 6518   ins_cost(4 * INSN_COST);
 6519   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6520 
 6521   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6522 
 6523   ins_pipe(iload_reg_mem);
 6524 %}
 6525 
 6526 // Load Byte (8 bit unsigned)
 6527 instruct loadUB(iRegINoSp dst, memory1 mem)
 6528 %{
 6529   match(Set dst (LoadUB mem));
 6530   predicate(!needs_acquiring_load(n));
 6531 
 6532   ins_cost(4 * INSN_COST);
 6533   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6534 
 6535   ins_encode(aarch64_enc_ldrb(dst, mem));
 6536 
 6537   ins_pipe(iload_reg_mem);
 6538 %}
 6539 
 6540 // Load Byte (8 bit unsigned) into long
 6541 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6542 %{
 6543   match(Set dst (ConvI2L (LoadUB mem)));
 6544   predicate(!needs_acquiring_load(n->in(1)));
 6545 
 6546   ins_cost(4 * INSN_COST);
 6547   format %{ "ldrb  $dst, $mem\t# byte" %}
 6548 
 6549   ins_encode(aarch64_enc_ldrb(dst, mem));
 6550 
 6551   ins_pipe(iload_reg_mem);
 6552 %}
 6553 
 6554 // Load Short (16 bit signed)
 6555 instruct loadS(iRegINoSp dst, memory2 mem)
 6556 %{
 6557   match(Set dst (LoadS mem));
 6558   predicate(!needs_acquiring_load(n));
 6559 
 6560   ins_cost(4 * INSN_COST);
 6561   format %{ "ldrshw  $dst, $mem\t# short" %}
 6562 
 6563   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6564 
 6565   ins_pipe(iload_reg_mem);
 6566 %}
 6567 
 6568 // Load Short (16 bit signed) into long
 6569 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6570 %{
 6571   match(Set dst (ConvI2L (LoadS mem)));
 6572   predicate(!needs_acquiring_load(n->in(1)));
 6573 
 6574   ins_cost(4 * INSN_COST);
 6575   format %{ "ldrsh  $dst, $mem\t# short" %}
 6576 
 6577   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6578 
 6579   ins_pipe(iload_reg_mem);
 6580 %}
 6581 
 6582 // Load Char (16 bit unsigned)
 6583 instruct loadUS(iRegINoSp dst, memory2 mem)
 6584 %{
 6585   match(Set dst (LoadUS mem));
 6586   predicate(!needs_acquiring_load(n));
 6587 
 6588   ins_cost(4 * INSN_COST);
 6589   format %{ "ldrh  $dst, $mem\t# short" %}
 6590 
 6591   ins_encode(aarch64_enc_ldrh(dst, mem));
 6592 
 6593   ins_pipe(iload_reg_mem);
 6594 %}
 6595 
 6596 // Load Short/Char (16 bit unsigned) into long
 6597 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6598 %{
 6599   match(Set dst (ConvI2L (LoadUS mem)));
 6600   predicate(!needs_acquiring_load(n->in(1)));
 6601 
 6602   ins_cost(4 * INSN_COST);
 6603   format %{ "ldrh  $dst, $mem\t# short" %}
 6604 
 6605   ins_encode(aarch64_enc_ldrh(dst, mem));
 6606 
 6607   ins_pipe(iload_reg_mem);
 6608 %}
 6609 
 6610 // Load Integer (32 bit signed)
 6611 instruct loadI(iRegINoSp dst, memory4 mem)
 6612 %{
 6613   match(Set dst (LoadI mem));
 6614   predicate(!needs_acquiring_load(n));
 6615 
 6616   ins_cost(4 * INSN_COST);
 6617   format %{ "ldrw  $dst, $mem\t# int" %}
 6618 
 6619   ins_encode(aarch64_enc_ldrw(dst, mem));
 6620 
 6621   ins_pipe(iload_reg_mem);
 6622 %}
 6623 
 6624 // Load Integer (32 bit signed) into long
 6625 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6626 %{
 6627   match(Set dst (ConvI2L (LoadI mem)));
 6628   predicate(!needs_acquiring_load(n->in(1)));
 6629 
 6630   ins_cost(4 * INSN_COST);
 6631   format %{ "ldrsw  $dst, $mem\t# int" %}
 6632 
 6633   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6634 
 6635   ins_pipe(iload_reg_mem);
 6636 %}
 6637 
 6638 // Load Integer (32 bit unsigned) into long
 6639 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6640 %{
 6641   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6642   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6643 
 6644   ins_cost(4 * INSN_COST);
 6645   format %{ "ldrw  $dst, $mem\t# int" %}
 6646 
 6647   ins_encode(aarch64_enc_ldrw(dst, mem));
 6648 
 6649   ins_pipe(iload_reg_mem);
 6650 %}
 6651 
 6652 // Load Long (64 bit signed)
 6653 instruct loadL(iRegLNoSp dst, memory8 mem)
 6654 %{
 6655   match(Set dst (LoadL mem));
 6656   predicate(!needs_acquiring_load(n));
 6657 
 6658   ins_cost(4 * INSN_COST);
 6659   format %{ "ldr  $dst, $mem\t# int" %}
 6660 
 6661   ins_encode(aarch64_enc_ldr(dst, mem));
 6662 
 6663   ins_pipe(iload_reg_mem);
 6664 %}
 6665 
 6666 // Load Range
 6667 instruct loadRange(iRegINoSp dst, memory4 mem)
 6668 %{
 6669   match(Set dst (LoadRange mem));
 6670 
 6671   ins_cost(4 * INSN_COST);
 6672   format %{ "ldrw  $dst, $mem\t# range" %}
 6673 
 6674   ins_encode(aarch64_enc_ldrw(dst, mem));
 6675 
 6676   ins_pipe(iload_reg_mem);
 6677 %}
 6678 
 6679 // Load Pointer
 6680 instruct loadP(iRegPNoSp dst, memory8 mem)
 6681 %{
 6682   match(Set dst (LoadP mem));
 6683   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6684 
 6685   ins_cost(4 * INSN_COST);
 6686   format %{ "ldr  $dst, $mem\t# ptr" %}
 6687 
 6688   ins_encode(aarch64_enc_ldr(dst, mem));
 6689 
 6690   ins_pipe(iload_reg_mem);
 6691 %}
 6692 
 6693 // Load Compressed Pointer
 6694 instruct loadN(iRegNNoSp dst, memory4 mem)
 6695 %{
 6696   match(Set dst (LoadN mem));
 6697   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6698 
 6699   ins_cost(4 * INSN_COST);
 6700   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6701 
 6702   ins_encode(aarch64_enc_ldrw(dst, mem));
 6703 
 6704   ins_pipe(iload_reg_mem);
 6705 %}
 6706 
 6707 // Load Klass Pointer
 6708 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6709 %{
 6710   match(Set dst (LoadKlass mem));
 6711   predicate(!needs_acquiring_load(n));
 6712 
 6713   ins_cost(4 * INSN_COST);
 6714   format %{ "ldr  $dst, $mem\t# class" %}
 6715 
 6716   ins_encode(aarch64_enc_ldr(dst, mem));
 6717 
 6718   ins_pipe(iload_reg_mem);
 6719 %}
 6720 
 6721 // Load Narrow Klass Pointer
 6722 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6723 %{
 6724   match(Set dst (LoadNKlass mem));
 6725   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6726 
 6727   ins_cost(4 * INSN_COST);
 6728   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6729 
 6730   ins_encode(aarch64_enc_ldrw(dst, mem));
 6731 
 6732   ins_pipe(iload_reg_mem);
 6733 %}
 6734 
 6735 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
 6736 %{
 6737   match(Set dst (LoadNKlass mem));
 6738   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6739 
 6740   ins_cost(4 * INSN_COST);
 6741   format %{
 6742     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6743     "lsrw  $dst, $dst, markWord::klass_shift_at_offset"
 6744   %}
 6745   ins_encode %{
 6746     // inlined aarch64_enc_ldrw
 6747     loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
 6748               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 6749     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
 6750   %}
 6751   ins_pipe(iload_reg_mem);
 6752 %}
 6753 
 6754 // Load Float
 6755 instruct loadF(vRegF dst, memory4 mem)
 6756 %{
 6757   match(Set dst (LoadF mem));
 6758   predicate(!needs_acquiring_load(n));
 6759 
 6760   ins_cost(4 * INSN_COST);
 6761   format %{ "ldrs  $dst, $mem\t# float" %}
 6762 
 6763   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6764 
 6765   ins_pipe(pipe_class_memory);
 6766 %}
 6767 
 6768 // Load Double
 6769 instruct loadD(vRegD dst, memory8 mem)
 6770 %{
 6771   match(Set dst (LoadD mem));
 6772   predicate(!needs_acquiring_load(n));
 6773 
 6774   ins_cost(4 * INSN_COST);
 6775   format %{ "ldrd  $dst, $mem\t# double" %}
 6776 
 6777   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6778 
 6779   ins_pipe(pipe_class_memory);
 6780 %}
 6781 
 6782 
 6783 // Load Int Constant
 6784 instruct loadConI(iRegINoSp dst, immI src)
 6785 %{
 6786   match(Set dst src);
 6787 
 6788   ins_cost(INSN_COST);
 6789   format %{ "mov $dst, $src\t# int" %}
 6790 
 6791   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6792 
 6793   ins_pipe(ialu_imm);
 6794 %}
 6795 
 6796 // Load Long Constant
 6797 instruct loadConL(iRegLNoSp dst, immL src)
 6798 %{
 6799   match(Set dst src);
 6800 
 6801   ins_cost(INSN_COST);
 6802   format %{ "mov $dst, $src\t# long" %}
 6803 
 6804   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6805 
 6806   ins_pipe(ialu_imm);
 6807 %}
 6808 
 6809 // Load Pointer Constant
 6810 
 6811 instruct loadConP(iRegPNoSp dst, immP con)
 6812 %{
 6813   match(Set dst con);
 6814 
 6815   ins_cost(INSN_COST * 4);
 6816   format %{
 6817     "mov  $dst, $con\t# ptr"
 6818   %}
 6819 
 6820   ins_encode(aarch64_enc_mov_p(dst, con));
 6821 
 6822   ins_pipe(ialu_imm);
 6823 %}
 6824 
 6825 // Load Null Pointer Constant
 6826 
 6827 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6828 %{
 6829   match(Set dst con);
 6830 
 6831   ins_cost(INSN_COST);
 6832   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6833 
 6834   ins_encode(aarch64_enc_mov_p0(dst, con));
 6835 
 6836   ins_pipe(ialu_imm);
 6837 %}
 6838 
 6839 // Load Pointer Constant One
 6840 
 6841 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6842 %{
 6843   match(Set dst con);
 6844 
 6845   ins_cost(INSN_COST);
 6846   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6847 
 6848   ins_encode(aarch64_enc_mov_p1(dst, con));
 6849 
 6850   ins_pipe(ialu_imm);
 6851 %}
 6852 
 6853 // Load Byte Map Base Constant
 6854 
 6855 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 6856 %{
 6857   match(Set dst con);
 6858 
 6859   ins_cost(INSN_COST);
 6860   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 6861 
 6862   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 6863 
 6864   ins_pipe(ialu_imm);
 6865 %}
 6866 
 6867 // Load Narrow Pointer Constant
 6868 
 6869 instruct loadConN(iRegNNoSp dst, immN con)
 6870 %{
 6871   match(Set dst con);
 6872 
 6873   ins_cost(INSN_COST * 4);
 6874   format %{ "mov  $dst, $con\t# compressed ptr" %}
 6875 
 6876   ins_encode(aarch64_enc_mov_n(dst, con));
 6877 
 6878   ins_pipe(ialu_imm);
 6879 %}
 6880 
 6881 // Load Narrow Null Pointer Constant
 6882 
 6883 instruct loadConN0(iRegNNoSp dst, immN0 con)
 6884 %{
 6885   match(Set dst con);
 6886 
 6887   ins_cost(INSN_COST);
 6888   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 6889 
 6890   ins_encode(aarch64_enc_mov_n0(dst, con));
 6891 
 6892   ins_pipe(ialu_imm);
 6893 %}
 6894 
 6895 // Load Narrow Klass Constant
 6896 
 6897 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 6898 %{
 6899   match(Set dst con);
 6900 
 6901   ins_cost(INSN_COST);
 6902   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 6903 
 6904   ins_encode(aarch64_enc_mov_nk(dst, con));
 6905 
 6906   ins_pipe(ialu_imm);
 6907 %}
 6908 
 6909 // Load Packed Float Constant
 6910 
 6911 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 6912   match(Set dst con);
 6913   ins_cost(INSN_COST * 4);
 6914   format %{ "fmovs  $dst, $con"%}
 6915   ins_encode %{
 6916     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 6917   %}
 6918 
 6919   ins_pipe(fp_imm_s);
 6920 %}
 6921 
 6922 // Load Float Constant
 6923 
 6924 instruct loadConF(vRegF dst, immF con) %{
 6925   match(Set dst con);
 6926 
 6927   ins_cost(INSN_COST * 4);
 6928 
 6929   format %{
 6930     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6931   %}
 6932 
 6933   ins_encode %{
 6934     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 6935   %}
 6936 
 6937   ins_pipe(fp_load_constant_s);
 6938 %}
 6939 
 6940 // Load Packed Double Constant
 6941 
 6942 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 6943   match(Set dst con);
 6944   ins_cost(INSN_COST);
 6945   format %{ "fmovd  $dst, $con"%}
 6946   ins_encode %{
 6947     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 6948   %}
 6949 
 6950   ins_pipe(fp_imm_d);
 6951 %}
 6952 
 6953 // Load Double Constant
 6954 
 6955 instruct loadConD(vRegD dst, immD con) %{
 6956   match(Set dst con);
 6957 
 6958   ins_cost(INSN_COST * 5);
 6959   format %{
 6960     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6961   %}
 6962 
 6963   ins_encode %{
 6964     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 6965   %}
 6966 
 6967   ins_pipe(fp_load_constant_d);
 6968 %}
 6969 
 6970 // Store Instructions
 6971 
 6972 // Store Byte
 6973 instruct storeB(iRegIorL2I src, memory1 mem)
 6974 %{
 6975   match(Set mem (StoreB mem src));
 6976   predicate(!needs_releasing_store(n));
 6977 
 6978   ins_cost(INSN_COST);
 6979   format %{ "strb  $src, $mem\t# byte" %}
 6980 
 6981   ins_encode(aarch64_enc_strb(src, mem));
 6982 
 6983   ins_pipe(istore_reg_mem);
 6984 %}
 6985 
 6986 
 6987 instruct storeimmB0(immI0 zero, memory1 mem)
 6988 %{
 6989   match(Set mem (StoreB mem zero));
 6990   predicate(!needs_releasing_store(n));
 6991 
 6992   ins_cost(INSN_COST);
 6993   format %{ "strb rscractch2, $mem\t# byte" %}
 6994 
 6995   ins_encode(aarch64_enc_strb0(mem));
 6996 
 6997   ins_pipe(istore_mem);
 6998 %}
 6999 
 7000 // Store Char/Short
 7001 instruct storeC(iRegIorL2I src, memory2 mem)
 7002 %{
 7003   match(Set mem (StoreC mem src));
 7004   predicate(!needs_releasing_store(n));
 7005 
 7006   ins_cost(INSN_COST);
 7007   format %{ "strh  $src, $mem\t# short" %}
 7008 
 7009   ins_encode(aarch64_enc_strh(src, mem));
 7010 
 7011   ins_pipe(istore_reg_mem);
 7012 %}
 7013 
 7014 instruct storeimmC0(immI0 zero, memory2 mem)
 7015 %{
 7016   match(Set mem (StoreC mem zero));
 7017   predicate(!needs_releasing_store(n));
 7018 
 7019   ins_cost(INSN_COST);
 7020   format %{ "strh  zr, $mem\t# short" %}
 7021 
 7022   ins_encode(aarch64_enc_strh0(mem));
 7023 
 7024   ins_pipe(istore_mem);
 7025 %}
 7026 
 7027 // Store Integer
 7028 
 7029 instruct storeI(iRegIorL2I src, memory4 mem)
 7030 %{
 7031   match(Set mem(StoreI mem src));
 7032   predicate(!needs_releasing_store(n));
 7033 
 7034   ins_cost(INSN_COST);
 7035   format %{ "strw  $src, $mem\t# int" %}
 7036 
 7037   ins_encode(aarch64_enc_strw(src, mem));
 7038 
 7039   ins_pipe(istore_reg_mem);
 7040 %}
 7041 
 7042 instruct storeimmI0(immI0 zero, memory4 mem)
 7043 %{
 7044   match(Set mem(StoreI mem zero));
 7045   predicate(!needs_releasing_store(n));
 7046 
 7047   ins_cost(INSN_COST);
 7048   format %{ "strw  zr, $mem\t# int" %}
 7049 
 7050   ins_encode(aarch64_enc_strw0(mem));
 7051 
 7052   ins_pipe(istore_mem);
 7053 %}
 7054 
 7055 // Store Long (64 bit signed)
 7056 instruct storeL(iRegL src, memory8 mem)
 7057 %{
 7058   match(Set mem (StoreL mem src));
 7059   predicate(!needs_releasing_store(n));
 7060 
 7061   ins_cost(INSN_COST);
 7062   format %{ "str  $src, $mem\t# int" %}
 7063 
 7064   ins_encode(aarch64_enc_str(src, mem));
 7065 
 7066   ins_pipe(istore_reg_mem);
 7067 %}
 7068 
 7069 // Store Long (64 bit signed)
 7070 instruct storeimmL0(immL0 zero, memory8 mem)
 7071 %{
 7072   match(Set mem (StoreL mem zero));
 7073   predicate(!needs_releasing_store(n));
 7074 
 7075   ins_cost(INSN_COST);
 7076   format %{ "str  zr, $mem\t# int" %}
 7077 
 7078   ins_encode(aarch64_enc_str0(mem));
 7079 
 7080   ins_pipe(istore_mem);
 7081 %}
 7082 
 7083 // Store Pointer
 7084 instruct storeP(iRegP src, memory8 mem)
 7085 %{
 7086   match(Set mem (StoreP mem src));
 7087   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7088 
 7089   ins_cost(INSN_COST);
 7090   format %{ "str  $src, $mem\t# ptr" %}
 7091 
 7092   ins_encode(aarch64_enc_str(src, mem));
 7093 
 7094   ins_pipe(istore_reg_mem);
 7095 %}
 7096 
 7097 // Store Pointer
 7098 instruct storeimmP0(immP0 zero, memory8 mem)
 7099 %{
 7100   match(Set mem (StoreP mem zero));
 7101   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7102 
 7103   ins_cost(INSN_COST);
 7104   format %{ "str zr, $mem\t# ptr" %}
 7105 
 7106   ins_encode(aarch64_enc_str0(mem));
 7107 
 7108   ins_pipe(istore_mem);
 7109 %}
 7110 
 7111 // Store Compressed Pointer
 7112 instruct storeN(iRegN src, memory4 mem)
 7113 %{
 7114   match(Set mem (StoreN mem src));
 7115   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7116 
 7117   ins_cost(INSN_COST);
 7118   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7119 
 7120   ins_encode(aarch64_enc_strw(src, mem));
 7121 
 7122   ins_pipe(istore_reg_mem);
 7123 %}
 7124 
 7125 instruct storeImmN0(immN0 zero, memory4 mem)
 7126 %{
 7127   match(Set mem (StoreN mem zero));
 7128   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7129 
 7130   ins_cost(INSN_COST);
 7131   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7132 
 7133   ins_encode(aarch64_enc_strw0(mem));
 7134 
 7135   ins_pipe(istore_mem);
 7136 %}
 7137 
 7138 // Store Float
 7139 instruct storeF(vRegF src, memory4 mem)
 7140 %{
 7141   match(Set mem (StoreF mem src));
 7142   predicate(!needs_releasing_store(n));
 7143 
 7144   ins_cost(INSN_COST);
 7145   format %{ "strs  $src, $mem\t# float" %}
 7146 
 7147   ins_encode( aarch64_enc_strs(src, mem) );
 7148 
 7149   ins_pipe(pipe_class_memory);
 7150 %}
 7151 
 7152 // TODO
 7153 // implement storeImmF0 and storeFImmPacked
 7154 
 7155 // Store Double
 7156 instruct storeD(vRegD src, memory8 mem)
 7157 %{
 7158   match(Set mem (StoreD mem src));
 7159   predicate(!needs_releasing_store(n));
 7160 
 7161   ins_cost(INSN_COST);
 7162   format %{ "strd  $src, $mem\t# double" %}
 7163 
 7164   ins_encode( aarch64_enc_strd(src, mem) );
 7165 
 7166   ins_pipe(pipe_class_memory);
 7167 %}
 7168 
 7169 // Store Compressed Klass Pointer
 7170 instruct storeNKlass(iRegN src, memory4 mem)
 7171 %{
 7172   predicate(!needs_releasing_store(n));
 7173   match(Set mem (StoreNKlass mem src));
 7174 
 7175   ins_cost(INSN_COST);
 7176   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7177 
 7178   ins_encode(aarch64_enc_strw(src, mem));
 7179 
 7180   ins_pipe(istore_reg_mem);
 7181 %}
 7182 
 7183 // TODO
 7184 // implement storeImmD0 and storeDImmPacked
 7185 
 7186 // prefetch instructions
 7187 // Must be safe to execute with invalid address (cannot fault).
 7188 
 7189 instruct prefetchalloc( memory8 mem ) %{
 7190   match(PrefetchAllocation mem);
 7191 
 7192   ins_cost(INSN_COST);
 7193   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7194 
 7195   ins_encode( aarch64_enc_prefetchw(mem) );
 7196 
 7197   ins_pipe(iload_prefetch);
 7198 %}
 7199 
 7200 //  ---------------- volatile loads and stores ----------------
 7201 
 7202 // Load Byte (8 bit signed)
 7203 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7204 %{
 7205   match(Set dst (LoadB mem));
 7206 
 7207   ins_cost(VOLATILE_REF_COST);
 7208   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7209 
 7210   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7211 
 7212   ins_pipe(pipe_serial);
 7213 %}
 7214 
 7215 // Load Byte (8 bit signed) into long
 7216 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7217 %{
 7218   match(Set dst (ConvI2L (LoadB mem)));
 7219 
 7220   ins_cost(VOLATILE_REF_COST);
 7221   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7222 
 7223   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7224 
 7225   ins_pipe(pipe_serial);
 7226 %}
 7227 
 7228 // Load Byte (8 bit unsigned)
 7229 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7230 %{
 7231   match(Set dst (LoadUB mem));
 7232 
 7233   ins_cost(VOLATILE_REF_COST);
 7234   format %{ "ldarb  $dst, $mem\t# byte" %}
 7235 
 7236   ins_encode(aarch64_enc_ldarb(dst, mem));
 7237 
 7238   ins_pipe(pipe_serial);
 7239 %}
 7240 
 7241 // Load Byte (8 bit unsigned) into long
 7242 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7243 %{
 7244   match(Set dst (ConvI2L (LoadUB mem)));
 7245 
 7246   ins_cost(VOLATILE_REF_COST);
 7247   format %{ "ldarb  $dst, $mem\t# byte" %}
 7248 
 7249   ins_encode(aarch64_enc_ldarb(dst, mem));
 7250 
 7251   ins_pipe(pipe_serial);
 7252 %}
 7253 
 7254 // Load Short (16 bit signed)
 7255 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7256 %{
 7257   match(Set dst (LoadS mem));
 7258 
 7259   ins_cost(VOLATILE_REF_COST);
 7260   format %{ "ldarshw  $dst, $mem\t# short" %}
 7261 
 7262   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7263 
 7264   ins_pipe(pipe_serial);
 7265 %}
 7266 
 7267 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7268 %{
 7269   match(Set dst (LoadUS mem));
 7270 
 7271   ins_cost(VOLATILE_REF_COST);
 7272   format %{ "ldarhw  $dst, $mem\t# short" %}
 7273 
 7274   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7275 
 7276   ins_pipe(pipe_serial);
 7277 %}
 7278 
 7279 // Load Short/Char (16 bit unsigned) into long
 7280 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7281 %{
 7282   match(Set dst (ConvI2L (LoadUS mem)));
 7283 
 7284   ins_cost(VOLATILE_REF_COST);
 7285   format %{ "ldarh  $dst, $mem\t# short" %}
 7286 
 7287   ins_encode(aarch64_enc_ldarh(dst, mem));
 7288 
 7289   ins_pipe(pipe_serial);
 7290 %}
 7291 
 7292 // Load Short/Char (16 bit signed) into long
 7293 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7294 %{
 7295   match(Set dst (ConvI2L (LoadS mem)));
 7296 
 7297   ins_cost(VOLATILE_REF_COST);
 7298   format %{ "ldarh  $dst, $mem\t# short" %}
 7299 
 7300   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7301 
 7302   ins_pipe(pipe_serial);
 7303 %}
 7304 
 7305 // Load Integer (32 bit signed)
 7306 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7307 %{
 7308   match(Set dst (LoadI mem));
 7309 
 7310   ins_cost(VOLATILE_REF_COST);
 7311   format %{ "ldarw  $dst, $mem\t# int" %}
 7312 
 7313   ins_encode(aarch64_enc_ldarw(dst, mem));
 7314 
 7315   ins_pipe(pipe_serial);
 7316 %}
 7317 
 7318 // Load Integer (32 bit unsigned) into long
 7319 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7320 %{
 7321   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7322 
 7323   ins_cost(VOLATILE_REF_COST);
 7324   format %{ "ldarw  $dst, $mem\t# int" %}
 7325 
 7326   ins_encode(aarch64_enc_ldarw(dst, mem));
 7327 
 7328   ins_pipe(pipe_serial);
 7329 %}
 7330 
 7331 // Load Long (64 bit signed)
 7332 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7333 %{
 7334   match(Set dst (LoadL mem));
 7335 
 7336   ins_cost(VOLATILE_REF_COST);
 7337   format %{ "ldar  $dst, $mem\t# int" %}
 7338 
 7339   ins_encode(aarch64_enc_ldar(dst, mem));
 7340 
 7341   ins_pipe(pipe_serial);
 7342 %}
 7343 
 7344 // Load Pointer
 7345 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7346 %{
 7347   match(Set dst (LoadP mem));
 7348   predicate(n->as_Load()->barrier_data() == 0);
 7349 
 7350   ins_cost(VOLATILE_REF_COST);
 7351   format %{ "ldar  $dst, $mem\t# ptr" %}
 7352 
 7353   ins_encode(aarch64_enc_ldar(dst, mem));
 7354 
 7355   ins_pipe(pipe_serial);
 7356 %}
 7357 
 7358 // Load Compressed Pointer
 7359 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7360 %{
 7361   match(Set dst (LoadN mem));
 7362   predicate(n->as_Load()->barrier_data() == 0);
 7363 
 7364   ins_cost(VOLATILE_REF_COST);
 7365   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7366 
 7367   ins_encode(aarch64_enc_ldarw(dst, mem));
 7368 
 7369   ins_pipe(pipe_serial);
 7370 %}
 7371 
 7372 // Load Float
 7373 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7374 %{
 7375   match(Set dst (LoadF mem));
 7376 
 7377   ins_cost(VOLATILE_REF_COST);
 7378   format %{ "ldars  $dst, $mem\t# float" %}
 7379 
 7380   ins_encode( aarch64_enc_fldars(dst, mem) );
 7381 
 7382   ins_pipe(pipe_serial);
 7383 %}
 7384 
 7385 // Load Double
 7386 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7387 %{
 7388   match(Set dst (LoadD mem));
 7389 
 7390   ins_cost(VOLATILE_REF_COST);
 7391   format %{ "ldard  $dst, $mem\t# double" %}
 7392 
 7393   ins_encode( aarch64_enc_fldard(dst, mem) );
 7394 
 7395   ins_pipe(pipe_serial);
 7396 %}
 7397 
 7398 // Store Byte
 7399 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7400 %{
 7401   match(Set mem (StoreB mem src));
 7402 
 7403   ins_cost(VOLATILE_REF_COST);
 7404   format %{ "stlrb  $src, $mem\t# byte" %}
 7405 
 7406   ins_encode(aarch64_enc_stlrb(src, mem));
 7407 
 7408   ins_pipe(pipe_class_memory);
 7409 %}
 7410 
 7411 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7412 %{
 7413   match(Set mem (StoreB mem zero));
 7414 
 7415   ins_cost(VOLATILE_REF_COST);
 7416   format %{ "stlrb  zr, $mem\t# byte" %}
 7417 
 7418   ins_encode(aarch64_enc_stlrb0(mem));
 7419 
 7420   ins_pipe(pipe_class_memory);
 7421 %}
 7422 
 7423 // Store Char/Short
 7424 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7425 %{
 7426   match(Set mem (StoreC mem src));
 7427 
 7428   ins_cost(VOLATILE_REF_COST);
 7429   format %{ "stlrh  $src, $mem\t# short" %}
 7430 
 7431   ins_encode(aarch64_enc_stlrh(src, mem));
 7432 
 7433   ins_pipe(pipe_class_memory);
 7434 %}
 7435 
 7436 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7437 %{
 7438   match(Set mem (StoreC mem zero));
 7439 
 7440   ins_cost(VOLATILE_REF_COST);
 7441   format %{ "stlrh  zr, $mem\t# short" %}
 7442 
 7443   ins_encode(aarch64_enc_stlrh0(mem));
 7444 
 7445   ins_pipe(pipe_class_memory);
 7446 %}
 7447 
 7448 // Store Integer
 7449 
 7450 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7451 %{
 7452   match(Set mem(StoreI mem src));
 7453 
 7454   ins_cost(VOLATILE_REF_COST);
 7455   format %{ "stlrw  $src, $mem\t# int" %}
 7456 
 7457   ins_encode(aarch64_enc_stlrw(src, mem));
 7458 
 7459   ins_pipe(pipe_class_memory);
 7460 %}
 7461 
 7462 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7463 %{
 7464   match(Set mem(StoreI mem zero));
 7465 
 7466   ins_cost(VOLATILE_REF_COST);
 7467   format %{ "stlrw  zr, $mem\t# int" %}
 7468 
 7469   ins_encode(aarch64_enc_stlrw0(mem));
 7470 
 7471   ins_pipe(pipe_class_memory);
 7472 %}
 7473 
 7474 // Store Long (64 bit signed)
 7475 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7476 %{
 7477   match(Set mem (StoreL mem src));
 7478 
 7479   ins_cost(VOLATILE_REF_COST);
 7480   format %{ "stlr  $src, $mem\t# int" %}
 7481 
 7482   ins_encode(aarch64_enc_stlr(src, mem));
 7483 
 7484   ins_pipe(pipe_class_memory);
 7485 %}
 7486 
 7487 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7488 %{
 7489   match(Set mem (StoreL mem zero));
 7490 
 7491   ins_cost(VOLATILE_REF_COST);
 7492   format %{ "stlr  zr, $mem\t# int" %}
 7493 
 7494   ins_encode(aarch64_enc_stlr0(mem));
 7495 
 7496   ins_pipe(pipe_class_memory);
 7497 %}
 7498 
 7499 // Store Pointer
 7500 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7501 %{
 7502   match(Set mem (StoreP mem src));
 7503   predicate(n->as_Store()->barrier_data() == 0);
 7504 
 7505   ins_cost(VOLATILE_REF_COST);
 7506   format %{ "stlr  $src, $mem\t# ptr" %}
 7507 
 7508   ins_encode(aarch64_enc_stlr(src, mem));
 7509 
 7510   ins_pipe(pipe_class_memory);
 7511 %}
 7512 
 7513 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7514 %{
 7515   match(Set mem (StoreP mem zero));
 7516   predicate(n->as_Store()->barrier_data() == 0);
 7517 
 7518   ins_cost(VOLATILE_REF_COST);
 7519   format %{ "stlr  zr, $mem\t# ptr" %}
 7520 
 7521   ins_encode(aarch64_enc_stlr0(mem));
 7522 
 7523   ins_pipe(pipe_class_memory);
 7524 %}
 7525 
 7526 // Store Compressed Pointer
 7527 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7528 %{
 7529   match(Set mem (StoreN mem src));
 7530   predicate(n->as_Store()->barrier_data() == 0);
 7531 
 7532   ins_cost(VOLATILE_REF_COST);
 7533   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7534 
 7535   ins_encode(aarch64_enc_stlrw(src, mem));
 7536 
 7537   ins_pipe(pipe_class_memory);
 7538 %}
 7539 
 7540 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7541 %{
 7542   match(Set mem (StoreN mem zero));
 7543   predicate(n->as_Store()->barrier_data() == 0);
 7544 
 7545   ins_cost(VOLATILE_REF_COST);
 7546   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7547 
 7548   ins_encode(aarch64_enc_stlrw0(mem));
 7549 
 7550   ins_pipe(pipe_class_memory);
 7551 %}
 7552 
 7553 // Store Float
 7554 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7555 %{
 7556   match(Set mem (StoreF mem src));
 7557 
 7558   ins_cost(VOLATILE_REF_COST);
 7559   format %{ "stlrs  $src, $mem\t# float" %}
 7560 
 7561   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7562 
 7563   ins_pipe(pipe_class_memory);
 7564 %}
 7565 
 7566 // TODO
 7567 // implement storeImmF0 and storeFImmPacked
 7568 
 7569 // Store Double
 7570 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7571 %{
 7572   match(Set mem (StoreD mem src));
 7573 
 7574   ins_cost(VOLATILE_REF_COST);
 7575   format %{ "stlrd  $src, $mem\t# double" %}
 7576 
 7577   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7578 
 7579   ins_pipe(pipe_class_memory);
 7580 %}
 7581 
 7582 //  ---------------- end of volatile loads and stores ----------------
 7583 
 7584 instruct cacheWB(indirect addr)
 7585 %{
 7586   predicate(VM_Version::supports_data_cache_line_flush());
 7587   match(CacheWB addr);
 7588 
 7589   ins_cost(100);
 7590   format %{"cache wb $addr" %}
 7591   ins_encode %{
 7592     assert($addr->index_position() < 0, "should be");
 7593     assert($addr$$disp == 0, "should be");
 7594     __ cache_wb(Address($addr$$base$$Register, 0));
 7595   %}
 7596   ins_pipe(pipe_slow); // XXX
 7597 %}
 7598 
 7599 instruct cacheWBPreSync()
 7600 %{
 7601   predicate(VM_Version::supports_data_cache_line_flush());
 7602   match(CacheWBPreSync);
 7603 
 7604   ins_cost(100);
 7605   format %{"cache wb presync" %}
 7606   ins_encode %{
 7607     __ cache_wbsync(true);
 7608   %}
 7609   ins_pipe(pipe_slow); // XXX
 7610 %}
 7611 
 7612 instruct cacheWBPostSync()
 7613 %{
 7614   predicate(VM_Version::supports_data_cache_line_flush());
 7615   match(CacheWBPostSync);
 7616 
 7617   ins_cost(100);
 7618   format %{"cache wb postsync" %}
 7619   ins_encode %{
 7620     __ cache_wbsync(false);
 7621   %}
 7622   ins_pipe(pipe_slow); // XXX
 7623 %}
 7624 
 7625 // ============================================================================
 7626 // BSWAP Instructions
 7627 
 7628 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7629   match(Set dst (ReverseBytesI src));
 7630 
 7631   ins_cost(INSN_COST);
 7632   format %{ "revw  $dst, $src" %}
 7633 
 7634   ins_encode %{
 7635     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7636   %}
 7637 
 7638   ins_pipe(ialu_reg);
 7639 %}
 7640 
 7641 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7642   match(Set dst (ReverseBytesL src));
 7643 
 7644   ins_cost(INSN_COST);
 7645   format %{ "rev  $dst, $src" %}
 7646 
 7647   ins_encode %{
 7648     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7649   %}
 7650 
 7651   ins_pipe(ialu_reg);
 7652 %}
 7653 
 7654 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7655   match(Set dst (ReverseBytesUS src));
 7656 
 7657   ins_cost(INSN_COST);
 7658   format %{ "rev16w  $dst, $src" %}
 7659 
 7660   ins_encode %{
 7661     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7662   %}
 7663 
 7664   ins_pipe(ialu_reg);
 7665 %}
 7666 
 7667 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7668   match(Set dst (ReverseBytesS src));
 7669 
 7670   ins_cost(INSN_COST);
 7671   format %{ "rev16w  $dst, $src\n\t"
 7672             "sbfmw $dst, $dst, #0, #15" %}
 7673 
 7674   ins_encode %{
 7675     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7676     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7677   %}
 7678 
 7679   ins_pipe(ialu_reg);
 7680 %}
 7681 
 7682 // ============================================================================
 7683 // Zero Count Instructions
 7684 
 7685 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7686   match(Set dst (CountLeadingZerosI src));
 7687 
 7688   ins_cost(INSN_COST);
 7689   format %{ "clzw  $dst, $src" %}
 7690   ins_encode %{
 7691     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7692   %}
 7693 
 7694   ins_pipe(ialu_reg);
 7695 %}
 7696 
 7697 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7698   match(Set dst (CountLeadingZerosL src));
 7699 
 7700   ins_cost(INSN_COST);
 7701   format %{ "clz   $dst, $src" %}
 7702   ins_encode %{
 7703     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7704   %}
 7705 
 7706   ins_pipe(ialu_reg);
 7707 %}
 7708 
 7709 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7710   match(Set dst (CountTrailingZerosI src));
 7711 
 7712   ins_cost(INSN_COST * 2);
 7713   format %{ "rbitw  $dst, $src\n\t"
 7714             "clzw   $dst, $dst" %}
 7715   ins_encode %{
 7716     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7717     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7718   %}
 7719 
 7720   ins_pipe(ialu_reg);
 7721 %}
 7722 
 7723 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7724   match(Set dst (CountTrailingZerosL src));
 7725 
 7726   ins_cost(INSN_COST * 2);
 7727   format %{ "rbit   $dst, $src\n\t"
 7728             "clz    $dst, $dst" %}
 7729   ins_encode %{
 7730     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7731     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7732   %}
 7733 
 7734   ins_pipe(ialu_reg);
 7735 %}
 7736 
 7737 //---------- Population Count Instructions -------------------------------------
 7738 //
 7739 
 7740 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7741   match(Set dst (PopCountI src));
 7742   effect(TEMP tmp);
 7743   ins_cost(INSN_COST * 13);
 7744 
 7745   format %{ "movw   $src, $src\n\t"
 7746             "mov    $tmp, $src\t# vector (1D)\n\t"
 7747             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7748             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7749             "mov    $dst, $tmp\t# vector (1D)" %}
 7750   ins_encode %{
 7751     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 7752     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7753     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7754     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7755     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7756   %}
 7757 
 7758   ins_pipe(pipe_class_default);
 7759 %}
 7760 
 7761 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7762   match(Set dst (PopCountI (LoadI mem)));
 7763   effect(TEMP tmp);
 7764   ins_cost(INSN_COST * 13);
 7765 
 7766   format %{ "ldrs   $tmp, $mem\n\t"
 7767             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7768             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7769             "mov    $dst, $tmp\t# vector (1D)" %}
 7770   ins_encode %{
 7771     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7772     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7773               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7774     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7775     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7776     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7777   %}
 7778 
 7779   ins_pipe(pipe_class_default);
 7780 %}
 7781 
 7782 // Note: Long.bitCount(long) returns an int.
 7783 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7784   match(Set dst (PopCountL src));
 7785   effect(TEMP tmp);
 7786   ins_cost(INSN_COST * 13);
 7787 
 7788   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7789             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7790             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7791             "mov    $dst, $tmp\t# vector (1D)" %}
 7792   ins_encode %{
 7793     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7794     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7795     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7796     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7797   %}
 7798 
 7799   ins_pipe(pipe_class_default);
 7800 %}
 7801 
 7802 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7803   match(Set dst (PopCountL (LoadL mem)));
 7804   effect(TEMP tmp);
 7805   ins_cost(INSN_COST * 13);
 7806 
 7807   format %{ "ldrd   $tmp, $mem\n\t"
 7808             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7809             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7810             "mov    $dst, $tmp\t# vector (1D)" %}
 7811   ins_encode %{
 7812     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7813     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7814               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7815     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7816     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7817     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7818   %}
 7819 
 7820   ins_pipe(pipe_class_default);
 7821 %}
 7822 
 7823 // ============================================================================
 7824 // VerifyVectorAlignment Instruction
 7825 
 7826 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7827   match(Set addr (VerifyVectorAlignment addr mask));
 7828   effect(KILL cr);
 7829   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7830   ins_encode %{
 7831     Label Lskip;
 7832     // check if masked bits of addr are zero
 7833     __ tst($addr$$Register, $mask$$constant);
 7834     __ br(Assembler::EQ, Lskip);
 7835     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7836     __ bind(Lskip);
 7837   %}
 7838   ins_pipe(pipe_slow);
 7839 %}
 7840 
 7841 // ============================================================================
 7842 // MemBar Instruction
 7843 
 7844 instruct load_fence() %{
 7845   match(LoadFence);
 7846   ins_cost(VOLATILE_REF_COST);
 7847 
 7848   format %{ "load_fence" %}
 7849 
 7850   ins_encode %{
 7851     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7852   %}
 7853   ins_pipe(pipe_serial);
 7854 %}
 7855 
 7856 instruct unnecessary_membar_acquire() %{
 7857   predicate(unnecessary_acquire(n));
 7858   match(MemBarAcquire);
 7859   ins_cost(0);
 7860 
 7861   format %{ "membar_acquire (elided)" %}
 7862 
 7863   ins_encode %{
 7864     __ block_comment("membar_acquire (elided)");
 7865   %}
 7866 
 7867   ins_pipe(pipe_class_empty);
 7868 %}
 7869 
 7870 instruct membar_acquire() %{
 7871   match(MemBarAcquire);
 7872   ins_cost(VOLATILE_REF_COST);
 7873 
 7874   format %{ "membar_acquire\n\t"
 7875             "dmb ishld" %}
 7876 
 7877   ins_encode %{
 7878     __ block_comment("membar_acquire");
 7879     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7880   %}
 7881 
 7882   ins_pipe(pipe_serial);
 7883 %}
 7884 
 7885 
 7886 instruct membar_acquire_lock() %{
 7887   match(MemBarAcquireLock);
 7888   ins_cost(VOLATILE_REF_COST);
 7889 
 7890   format %{ "membar_acquire_lock (elided)" %}
 7891 
 7892   ins_encode %{
 7893     __ block_comment("membar_acquire_lock (elided)");
 7894   %}
 7895 
 7896   ins_pipe(pipe_serial);
 7897 %}
 7898 
 7899 instruct store_fence() %{
 7900   match(StoreFence);
 7901   ins_cost(VOLATILE_REF_COST);
 7902 
 7903   format %{ "store_fence" %}
 7904 
 7905   ins_encode %{
 7906     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 7907   %}
 7908   ins_pipe(pipe_serial);
 7909 %}
 7910 
 7911 instruct unnecessary_membar_release() %{
 7912   predicate(unnecessary_release(n));
 7913   match(MemBarRelease);
 7914   ins_cost(0);
 7915 
 7916   format %{ "membar_release (elided)" %}
 7917 
 7918   ins_encode %{
 7919     __ block_comment("membar_release (elided)");
 7920   %}
 7921   ins_pipe(pipe_serial);
 7922 %}
 7923 
 7924 instruct membar_release() %{
 7925   match(MemBarRelease);
 7926   ins_cost(VOLATILE_REF_COST);
 7927 
 7928   format %{ "membar_release\n\t"
 7929             "dmb ishst\n\tdmb ishld" %}
 7930 
 7931   ins_encode %{
 7932     __ block_comment("membar_release");
 7933     // These will be merged if AlwaysMergeDMB is enabled.
 7934     __ membar(Assembler::StoreStore);
 7935     __ membar(Assembler::LoadStore);
 7936   %}
 7937   ins_pipe(pipe_serial);
 7938 %}
 7939 
 7940 instruct membar_storestore() %{
 7941   match(MemBarStoreStore);
 7942   match(StoreStoreFence);
 7943   ins_cost(VOLATILE_REF_COST);
 7944 
 7945   format %{ "MEMBAR-store-store" %}
 7946 
 7947   ins_encode %{
 7948     __ membar(Assembler::StoreStore);
 7949   %}
 7950   ins_pipe(pipe_serial);
 7951 %}
 7952 
 7953 instruct membar_release_lock() %{
 7954   match(MemBarReleaseLock);
 7955   ins_cost(VOLATILE_REF_COST);
 7956 
 7957   format %{ "membar_release_lock (elided)" %}
 7958 
 7959   ins_encode %{
 7960     __ block_comment("membar_release_lock (elided)");
 7961   %}
 7962 
 7963   ins_pipe(pipe_serial);
 7964 %}
 7965 
 7966 instruct unnecessary_membar_volatile() %{
 7967   predicate(unnecessary_volatile(n));
 7968   match(MemBarVolatile);
 7969   ins_cost(0);
 7970 
 7971   format %{ "membar_volatile (elided)" %}
 7972 
 7973   ins_encode %{
 7974     __ block_comment("membar_volatile (elided)");
 7975   %}
 7976 
 7977   ins_pipe(pipe_serial);
 7978 %}
 7979 
 7980 instruct membar_volatile() %{
 7981   match(MemBarVolatile);
 7982   ins_cost(VOLATILE_REF_COST*100);
 7983 
 7984   format %{ "membar_volatile\n\t"
 7985              "dmb ish"%}
 7986 
 7987   ins_encode %{
 7988     __ block_comment("membar_volatile");
 7989     __ membar(Assembler::StoreLoad);
 7990   %}
 7991 
 7992   ins_pipe(pipe_serial);
 7993 %}
 7994 
 7995 // ============================================================================
 7996 // Cast/Convert Instructions
 7997 
 7998 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 7999   match(Set dst (CastX2P src));
 8000 
 8001   ins_cost(INSN_COST);
 8002   format %{ "mov $dst, $src\t# long -> ptr" %}
 8003 
 8004   ins_encode %{
 8005     if ($dst$$reg != $src$$reg) {
 8006       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8007     }
 8008   %}
 8009 
 8010   ins_pipe(ialu_reg);
 8011 %}
 8012 
 8013 instruct castI2N(iRegNNoSp dst, iRegI src) %{
 8014   match(Set dst (CastI2N src));
 8015 
 8016   ins_cost(INSN_COST);
 8017   format %{ "mov $dst, $src\t# int -> narrow ptr" %}
 8018 
 8019   ins_encode %{
 8020     if ($dst$$reg != $src$$reg) {
 8021       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8022     }
 8023   %}
 8024 
 8025   ins_pipe(ialu_reg);
 8026 %}
 8027 
 8028 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8029   match(Set dst (CastP2X src));
 8030 
 8031   ins_cost(INSN_COST);
 8032   format %{ "mov $dst, $src\t# ptr -> long" %}
 8033 
 8034   ins_encode %{
 8035     if ($dst$$reg != $src$$reg) {
 8036       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8037     }
 8038   %}
 8039 
 8040   ins_pipe(ialu_reg);
 8041 %}
 8042 
 8043 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8044   match(Set dst (CastP2X src));
 8045 
 8046   ins_cost(INSN_COST);
 8047   format %{ "mov $dst, $src\t# ptr -> long" %}
 8048 
 8049   ins_encode %{
 8050     if ($dst$$reg != $src$$reg) {
 8051       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8052     }
 8053   %}
 8054 
 8055   ins_pipe(ialu_reg);
 8056 %}
 8057 
 8058 // Convert oop into int for vectors alignment masking
 8059 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8060   match(Set dst (ConvL2I (CastP2X src)));
 8061 
 8062   ins_cost(INSN_COST);
 8063   format %{ "movw $dst, $src\t# ptr -> int" %}
 8064   ins_encode %{
 8065     __ movw($dst$$Register, $src$$Register);
 8066   %}
 8067 
 8068   ins_pipe(ialu_reg);
 8069 %}
 8070 
 8071 // Convert compressed oop into int for vectors alignment masking
 8072 // in case of 32bit oops (heap < 4Gb).
 8073 instruct convN2I(iRegINoSp dst, iRegN src)
 8074 %{
 8075   predicate(CompressedOops::shift() == 0);
 8076   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8077 
 8078   ins_cost(INSN_COST);
 8079   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8080   ins_encode %{
 8081     __ movw($dst$$Register, $src$$Register);
 8082   %}
 8083 
 8084   ins_pipe(ialu_reg);
 8085 %}
 8086 
 8087 
 8088 // Convert oop pointer into compressed form
 8089 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8090   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8091   match(Set dst (EncodeP src));
 8092   effect(KILL cr);
 8093   ins_cost(INSN_COST * 3);
 8094   format %{ "encode_heap_oop $dst, $src" %}
 8095   ins_encode %{
 8096     Register s = $src$$Register;
 8097     Register d = $dst$$Register;
 8098     __ encode_heap_oop(d, s);
 8099   %}
 8100   ins_pipe(ialu_reg);
 8101 %}
 8102 
 8103 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8104   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8105   match(Set dst (EncodeP src));
 8106   ins_cost(INSN_COST * 3);
 8107   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8108   ins_encode %{
 8109     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8110   %}
 8111   ins_pipe(ialu_reg);
 8112 %}
 8113 
 8114 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8115   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8116             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8117   match(Set dst (DecodeN src));
 8118   ins_cost(INSN_COST * 3);
 8119   format %{ "decode_heap_oop $dst, $src" %}
 8120   ins_encode %{
 8121     Register s = $src$$Register;
 8122     Register d = $dst$$Register;
 8123     __ decode_heap_oop(d, s);
 8124   %}
 8125   ins_pipe(ialu_reg);
 8126 %}
 8127 
 8128 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8129   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8130             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8131   match(Set dst (DecodeN src));
 8132   ins_cost(INSN_COST * 3);
 8133   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8134   ins_encode %{
 8135     Register s = $src$$Register;
 8136     Register d = $dst$$Register;
 8137     __ decode_heap_oop_not_null(d, s);
 8138   %}
 8139   ins_pipe(ialu_reg);
 8140 %}
 8141 
 8142 // n.b. AArch64 implementations of encode_klass_not_null and
 8143 // decode_klass_not_null do not modify the flags register so, unlike
 8144 // Intel, we don't kill CR as a side effect here
 8145 
 8146 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8147   match(Set dst (EncodePKlass src));
 8148 
 8149   ins_cost(INSN_COST * 3);
 8150   format %{ "encode_klass_not_null $dst,$src" %}
 8151 
 8152   ins_encode %{
 8153     Register src_reg = as_Register($src$$reg);
 8154     Register dst_reg = as_Register($dst$$reg);
 8155     __ encode_klass_not_null(dst_reg, src_reg);
 8156   %}
 8157 
 8158    ins_pipe(ialu_reg);
 8159 %}
 8160 
 8161 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8162   match(Set dst (DecodeNKlass src));
 8163 
 8164   ins_cost(INSN_COST * 3);
 8165   format %{ "decode_klass_not_null $dst,$src" %}
 8166 
 8167   ins_encode %{
 8168     Register src_reg = as_Register($src$$reg);
 8169     Register dst_reg = as_Register($dst$$reg);
 8170     if (dst_reg != src_reg) {
 8171       __ decode_klass_not_null(dst_reg, src_reg);
 8172     } else {
 8173       __ decode_klass_not_null(dst_reg);
 8174     }
 8175   %}
 8176 
 8177    ins_pipe(ialu_reg);
 8178 %}
 8179 
 8180 instruct checkCastPP(iRegPNoSp dst)
 8181 %{
 8182   match(Set dst (CheckCastPP dst));
 8183 
 8184   size(0);
 8185   format %{ "# checkcastPP of $dst" %}
 8186   ins_encode(/* empty encoding */);
 8187   ins_pipe(pipe_class_empty);
 8188 %}
 8189 
 8190 instruct castPP(iRegPNoSp dst)
 8191 %{
 8192   match(Set dst (CastPP dst));
 8193 
 8194   size(0);
 8195   format %{ "# castPP of $dst" %}
 8196   ins_encode(/* empty encoding */);
 8197   ins_pipe(pipe_class_empty);
 8198 %}
 8199 
 8200 instruct castII(iRegI dst)
 8201 %{
 8202   match(Set dst (CastII dst));
 8203 
 8204   size(0);
 8205   format %{ "# castII of $dst" %}
 8206   ins_encode(/* empty encoding */);
 8207   ins_cost(0);
 8208   ins_pipe(pipe_class_empty);
 8209 %}
 8210 
 8211 instruct castLL(iRegL dst)
 8212 %{
 8213   match(Set dst (CastLL dst));
 8214 
 8215   size(0);
 8216   format %{ "# castLL of $dst" %}
 8217   ins_encode(/* empty encoding */);
 8218   ins_cost(0);
 8219   ins_pipe(pipe_class_empty);
 8220 %}
 8221 
 8222 instruct castFF(vRegF dst)
 8223 %{
 8224   match(Set dst (CastFF dst));
 8225 
 8226   size(0);
 8227   format %{ "# castFF of $dst" %}
 8228   ins_encode(/* empty encoding */);
 8229   ins_cost(0);
 8230   ins_pipe(pipe_class_empty);
 8231 %}
 8232 
 8233 instruct castDD(vRegD dst)
 8234 %{
 8235   match(Set dst (CastDD dst));
 8236 
 8237   size(0);
 8238   format %{ "# castDD of $dst" %}
 8239   ins_encode(/* empty encoding */);
 8240   ins_cost(0);
 8241   ins_pipe(pipe_class_empty);
 8242 %}
 8243 
 8244 instruct castVV(vReg dst)
 8245 %{
 8246   match(Set dst (CastVV dst));
 8247 
 8248   size(0);
 8249   format %{ "# castVV of $dst" %}
 8250   ins_encode(/* empty encoding */);
 8251   ins_cost(0);
 8252   ins_pipe(pipe_class_empty);
 8253 %}
 8254 
 8255 instruct castVVMask(pRegGov dst)
 8256 %{
 8257   match(Set dst (CastVV dst));
 8258 
 8259   size(0);
 8260   format %{ "# castVV of $dst" %}
 8261   ins_encode(/* empty encoding */);
 8262   ins_cost(0);
 8263   ins_pipe(pipe_class_empty);
 8264 %}
 8265 
 8266 // ============================================================================
 8267 // Atomic operation instructions
 8268 //
 8269 
 8270 // standard CompareAndSwapX when we are using barriers
 8271 // these have higher priority than the rules selected by a predicate
 8272 
 8273 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8274 // can't match them
 8275 
 8276 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8277 
 8278   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8279   ins_cost(2 * VOLATILE_REF_COST);
 8280 
 8281   effect(KILL cr);
 8282 
 8283   format %{
 8284     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8285     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8286   %}
 8287 
 8288   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8289             aarch64_enc_cset_eq(res));
 8290 
 8291   ins_pipe(pipe_slow);
 8292 %}
 8293 
 8294 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8295 
 8296   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8297   ins_cost(2 * VOLATILE_REF_COST);
 8298 
 8299   effect(KILL cr);
 8300 
 8301   format %{
 8302     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8303     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8304   %}
 8305 
 8306   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8307             aarch64_enc_cset_eq(res));
 8308 
 8309   ins_pipe(pipe_slow);
 8310 %}
 8311 
 8312 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8313 
 8314   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8315   ins_cost(2 * VOLATILE_REF_COST);
 8316 
 8317   effect(KILL cr);
 8318 
 8319  format %{
 8320     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8321     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8322  %}
 8323 
 8324  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8325             aarch64_enc_cset_eq(res));
 8326 
 8327   ins_pipe(pipe_slow);
 8328 %}
 8329 
 8330 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8331 
 8332   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8333   ins_cost(2 * VOLATILE_REF_COST);
 8334 
 8335   effect(KILL cr);
 8336 
 8337  format %{
 8338     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8339     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8340  %}
 8341 
 8342  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8343             aarch64_enc_cset_eq(res));
 8344 
 8345   ins_pipe(pipe_slow);
 8346 %}
 8347 
 8348 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8349 
 8350   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8351   predicate(n->as_LoadStore()->barrier_data() == 0);
 8352   ins_cost(2 * VOLATILE_REF_COST);
 8353 
 8354   effect(KILL cr);
 8355 
 8356  format %{
 8357     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8358     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8359  %}
 8360 
 8361  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8362             aarch64_enc_cset_eq(res));
 8363 
 8364   ins_pipe(pipe_slow);
 8365 %}
 8366 
 8367 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8368 
 8369   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8370   predicate(n->as_LoadStore()->barrier_data() == 0);
 8371   ins_cost(2 * VOLATILE_REF_COST);
 8372 
 8373   effect(KILL cr);
 8374 
 8375  format %{
 8376     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8377     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8378  %}
 8379 
 8380  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8381             aarch64_enc_cset_eq(res));
 8382 
 8383   ins_pipe(pipe_slow);
 8384 %}
 8385 
 8386 // alternative CompareAndSwapX when we are eliding barriers
 8387 
 8388 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8389 
 8390   predicate(needs_acquiring_load_exclusive(n));
 8391   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8392   ins_cost(VOLATILE_REF_COST);
 8393 
 8394   effect(KILL cr);
 8395 
 8396   format %{
 8397     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8398     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8399   %}
 8400 
 8401   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8402             aarch64_enc_cset_eq(res));
 8403 
 8404   ins_pipe(pipe_slow);
 8405 %}
 8406 
 8407 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8408 
 8409   predicate(needs_acquiring_load_exclusive(n));
 8410   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8411   ins_cost(VOLATILE_REF_COST);
 8412 
 8413   effect(KILL cr);
 8414 
 8415   format %{
 8416     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8417     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8418   %}
 8419 
 8420   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8421             aarch64_enc_cset_eq(res));
 8422 
 8423   ins_pipe(pipe_slow);
 8424 %}
 8425 
 8426 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8427 
 8428   predicate(needs_acquiring_load_exclusive(n));
 8429   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8430   ins_cost(VOLATILE_REF_COST);
 8431 
 8432   effect(KILL cr);
 8433 
 8434  format %{
 8435     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8436     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8437  %}
 8438 
 8439  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8440             aarch64_enc_cset_eq(res));
 8441 
 8442   ins_pipe(pipe_slow);
 8443 %}
 8444 
 8445 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8446 
 8447   predicate(needs_acquiring_load_exclusive(n));
 8448   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8449   ins_cost(VOLATILE_REF_COST);
 8450 
 8451   effect(KILL cr);
 8452 
 8453  format %{
 8454     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8455     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8456  %}
 8457 
 8458  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8459             aarch64_enc_cset_eq(res));
 8460 
 8461   ins_pipe(pipe_slow);
 8462 %}
 8463 
 8464 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8465 
 8466   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8467   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8468   ins_cost(VOLATILE_REF_COST);
 8469 
 8470   effect(KILL cr);
 8471 
 8472  format %{
 8473     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8474     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8475  %}
 8476 
 8477  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8478             aarch64_enc_cset_eq(res));
 8479 
 8480   ins_pipe(pipe_slow);
 8481 %}
 8482 
 8483 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8484 
 8485   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8486   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8487   ins_cost(VOLATILE_REF_COST);
 8488 
 8489   effect(KILL cr);
 8490 
 8491  format %{
 8492     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8493     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8494  %}
 8495 
 8496  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8497             aarch64_enc_cset_eq(res));
 8498 
 8499   ins_pipe(pipe_slow);
 8500 %}
 8501 
 8502 
 8503 // ---------------------------------------------------------------------
 8504 
 8505 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8506 
 8507 // Sundry CAS operations.  Note that release is always true,
 8508 // regardless of the memory ordering of the CAS.  This is because we
 8509 // need the volatile case to be sequentially consistent but there is
 8510 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8511 // can't check the type of memory ordering here, so we always emit a
 8512 // STLXR.
 8513 
 8514 // This section is generated from cas.m4
 8515 
 8516 
 8517 // This pattern is generated automatically from cas.m4.
 8518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8519 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8520   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8521   ins_cost(2 * VOLATILE_REF_COST);
 8522   effect(TEMP_DEF res, KILL cr);
 8523   format %{
 8524     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8525   %}
 8526   ins_encode %{
 8527     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8528                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8529                /*weak*/ false, $res$$Register);
 8530     __ sxtbw($res$$Register, $res$$Register);
 8531   %}
 8532   ins_pipe(pipe_slow);
 8533 %}
 8534 
 8535 // This pattern is generated automatically from cas.m4.
 8536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8537 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8538   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8539   ins_cost(2 * VOLATILE_REF_COST);
 8540   effect(TEMP_DEF res, KILL cr);
 8541   format %{
 8542     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8543   %}
 8544   ins_encode %{
 8545     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8546                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8547                /*weak*/ false, $res$$Register);
 8548     __ sxthw($res$$Register, $res$$Register);
 8549   %}
 8550   ins_pipe(pipe_slow);
 8551 %}
 8552 
 8553 // This pattern is generated automatically from cas.m4.
 8554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8555 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8556   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8557   ins_cost(2 * VOLATILE_REF_COST);
 8558   effect(TEMP_DEF res, KILL cr);
 8559   format %{
 8560     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8561   %}
 8562   ins_encode %{
 8563     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8564                Assembler::word, /*acquire*/ false, /*release*/ true,
 8565                /*weak*/ false, $res$$Register);
 8566   %}
 8567   ins_pipe(pipe_slow);
 8568 %}
 8569 
 8570 // This pattern is generated automatically from cas.m4.
 8571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8572 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8573   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8574   ins_cost(2 * VOLATILE_REF_COST);
 8575   effect(TEMP_DEF res, KILL cr);
 8576   format %{
 8577     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8578   %}
 8579   ins_encode %{
 8580     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8581                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8582                /*weak*/ false, $res$$Register);
 8583   %}
 8584   ins_pipe(pipe_slow);
 8585 %}
 8586 
 8587 // This pattern is generated automatically from cas.m4.
 8588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8589 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8590   predicate(n->as_LoadStore()->barrier_data() == 0);
 8591   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8592   ins_cost(2 * VOLATILE_REF_COST);
 8593   effect(TEMP_DEF res, KILL cr);
 8594   format %{
 8595     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8596   %}
 8597   ins_encode %{
 8598     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8599                Assembler::word, /*acquire*/ false, /*release*/ true,
 8600                /*weak*/ false, $res$$Register);
 8601   %}
 8602   ins_pipe(pipe_slow);
 8603 %}
 8604 
 8605 // This pattern is generated automatically from cas.m4.
 8606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8607 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8608   predicate(n->as_LoadStore()->barrier_data() == 0);
 8609   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8610   ins_cost(2 * VOLATILE_REF_COST);
 8611   effect(TEMP_DEF res, KILL cr);
 8612   format %{
 8613     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8614   %}
 8615   ins_encode %{
 8616     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8617                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8618                /*weak*/ false, $res$$Register);
 8619   %}
 8620   ins_pipe(pipe_slow);
 8621 %}
 8622 
 8623 // This pattern is generated automatically from cas.m4.
 8624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8625 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8626   predicate(needs_acquiring_load_exclusive(n));
 8627   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8628   ins_cost(VOLATILE_REF_COST);
 8629   effect(TEMP_DEF res, KILL cr);
 8630   format %{
 8631     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8632   %}
 8633   ins_encode %{
 8634     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8635                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8636                /*weak*/ false, $res$$Register);
 8637     __ sxtbw($res$$Register, $res$$Register);
 8638   %}
 8639   ins_pipe(pipe_slow);
 8640 %}
 8641 
 8642 // This pattern is generated automatically from cas.m4.
 8643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8644 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8645   predicate(needs_acquiring_load_exclusive(n));
 8646   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8647   ins_cost(VOLATILE_REF_COST);
 8648   effect(TEMP_DEF res, KILL cr);
 8649   format %{
 8650     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8651   %}
 8652   ins_encode %{
 8653     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8654                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8655                /*weak*/ false, $res$$Register);
 8656     __ sxthw($res$$Register, $res$$Register);
 8657   %}
 8658   ins_pipe(pipe_slow);
 8659 %}
 8660 
 8661 // This pattern is generated automatically from cas.m4.
 8662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8663 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8664   predicate(needs_acquiring_load_exclusive(n));
 8665   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8666   ins_cost(VOLATILE_REF_COST);
 8667   effect(TEMP_DEF res, KILL cr);
 8668   format %{
 8669     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8670   %}
 8671   ins_encode %{
 8672     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8673                Assembler::word, /*acquire*/ true, /*release*/ true,
 8674                /*weak*/ false, $res$$Register);
 8675   %}
 8676   ins_pipe(pipe_slow);
 8677 %}
 8678 
 8679 // This pattern is generated automatically from cas.m4.
 8680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8681 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8682   predicate(needs_acquiring_load_exclusive(n));
 8683   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8684   ins_cost(VOLATILE_REF_COST);
 8685   effect(TEMP_DEF res, KILL cr);
 8686   format %{
 8687     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8688   %}
 8689   ins_encode %{
 8690     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8691                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8692                /*weak*/ false, $res$$Register);
 8693   %}
 8694   ins_pipe(pipe_slow);
 8695 %}
 8696 
 8697 // This pattern is generated automatically from cas.m4.
 8698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8699 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8700   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8701   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8702   ins_cost(VOLATILE_REF_COST);
 8703   effect(TEMP_DEF res, KILL cr);
 8704   format %{
 8705     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8706   %}
 8707   ins_encode %{
 8708     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8709                Assembler::word, /*acquire*/ true, /*release*/ true,
 8710                /*weak*/ false, $res$$Register);
 8711   %}
 8712   ins_pipe(pipe_slow);
 8713 %}
 8714 
 8715 // This pattern is generated automatically from cas.m4.
 8716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8717 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8718   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8719   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8720   ins_cost(VOLATILE_REF_COST);
 8721   effect(TEMP_DEF res, KILL cr);
 8722   format %{
 8723     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8724   %}
 8725   ins_encode %{
 8726     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8727                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8728                /*weak*/ false, $res$$Register);
 8729   %}
 8730   ins_pipe(pipe_slow);
 8731 %}
 8732 
 8733 // This pattern is generated automatically from cas.m4.
 8734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8735 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8736   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8737   ins_cost(2 * VOLATILE_REF_COST);
 8738   effect(KILL cr);
 8739   format %{
 8740     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8741     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8742   %}
 8743   ins_encode %{
 8744     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8745                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8746                /*weak*/ true, noreg);
 8747     __ csetw($res$$Register, Assembler::EQ);
 8748   %}
 8749   ins_pipe(pipe_slow);
 8750 %}
 8751 
 8752 // This pattern is generated automatically from cas.m4.
 8753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8754 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8755   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8756   ins_cost(2 * VOLATILE_REF_COST);
 8757   effect(KILL cr);
 8758   format %{
 8759     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8760     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8761   %}
 8762   ins_encode %{
 8763     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8764                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8765                /*weak*/ true, noreg);
 8766     __ csetw($res$$Register, Assembler::EQ);
 8767   %}
 8768   ins_pipe(pipe_slow);
 8769 %}
 8770 
 8771 // This pattern is generated automatically from cas.m4.
 8772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8773 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8774   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8775   ins_cost(2 * VOLATILE_REF_COST);
 8776   effect(KILL cr);
 8777   format %{
 8778     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8779     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8780   %}
 8781   ins_encode %{
 8782     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8783                Assembler::word, /*acquire*/ false, /*release*/ true,
 8784                /*weak*/ true, noreg);
 8785     __ csetw($res$$Register, Assembler::EQ);
 8786   %}
 8787   ins_pipe(pipe_slow);
 8788 %}
 8789 
 8790 // This pattern is generated automatically from cas.m4.
 8791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8792 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8793   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8794   ins_cost(2 * VOLATILE_REF_COST);
 8795   effect(KILL cr);
 8796   format %{
 8797     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8798     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8799   %}
 8800   ins_encode %{
 8801     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8802                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8803                /*weak*/ true, noreg);
 8804     __ csetw($res$$Register, Assembler::EQ);
 8805   %}
 8806   ins_pipe(pipe_slow);
 8807 %}
 8808 
 8809 // This pattern is generated automatically from cas.m4.
 8810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8811 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8812   predicate(n->as_LoadStore()->barrier_data() == 0);
 8813   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8814   ins_cost(2 * VOLATILE_REF_COST);
 8815   effect(KILL cr);
 8816   format %{
 8817     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8818     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8819   %}
 8820   ins_encode %{
 8821     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8822                Assembler::word, /*acquire*/ false, /*release*/ true,
 8823                /*weak*/ true, noreg);
 8824     __ csetw($res$$Register, Assembler::EQ);
 8825   %}
 8826   ins_pipe(pipe_slow);
 8827 %}
 8828 
 8829 // This pattern is generated automatically from cas.m4.
 8830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8831 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8832   predicate(n->as_LoadStore()->barrier_data() == 0);
 8833   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8834   ins_cost(2 * VOLATILE_REF_COST);
 8835   effect(KILL cr);
 8836   format %{
 8837     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8838     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8839   %}
 8840   ins_encode %{
 8841     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8842                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8843                /*weak*/ true, noreg);
 8844     __ csetw($res$$Register, Assembler::EQ);
 8845   %}
 8846   ins_pipe(pipe_slow);
 8847 %}
 8848 
 8849 // This pattern is generated automatically from cas.m4.
 8850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8851 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8852   predicate(needs_acquiring_load_exclusive(n));
 8853   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8854   ins_cost(VOLATILE_REF_COST);
 8855   effect(KILL cr);
 8856   format %{
 8857     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8858     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8859   %}
 8860   ins_encode %{
 8861     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8862                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8863                /*weak*/ true, noreg);
 8864     __ csetw($res$$Register, Assembler::EQ);
 8865   %}
 8866   ins_pipe(pipe_slow);
 8867 %}
 8868 
 8869 // This pattern is generated automatically from cas.m4.
 8870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8871 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8872   predicate(needs_acquiring_load_exclusive(n));
 8873   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8874   ins_cost(VOLATILE_REF_COST);
 8875   effect(KILL cr);
 8876   format %{
 8877     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8878     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8879   %}
 8880   ins_encode %{
 8881     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8882                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8883                /*weak*/ true, noreg);
 8884     __ csetw($res$$Register, Assembler::EQ);
 8885   %}
 8886   ins_pipe(pipe_slow);
 8887 %}
 8888 
 8889 // This pattern is generated automatically from cas.m4.
 8890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8891 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8892   predicate(needs_acquiring_load_exclusive(n));
 8893   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8894   ins_cost(VOLATILE_REF_COST);
 8895   effect(KILL cr);
 8896   format %{
 8897     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8898     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8899   %}
 8900   ins_encode %{
 8901     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8902                Assembler::word, /*acquire*/ true, /*release*/ true,
 8903                /*weak*/ true, noreg);
 8904     __ csetw($res$$Register, Assembler::EQ);
 8905   %}
 8906   ins_pipe(pipe_slow);
 8907 %}
 8908 
 8909 // This pattern is generated automatically from cas.m4.
 8910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8911 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8912   predicate(needs_acquiring_load_exclusive(n));
 8913   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8914   ins_cost(VOLATILE_REF_COST);
 8915   effect(KILL cr);
 8916   format %{
 8917     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8918     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8919   %}
 8920   ins_encode %{
 8921     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8922                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8923                /*weak*/ true, noreg);
 8924     __ csetw($res$$Register, Assembler::EQ);
 8925   %}
 8926   ins_pipe(pipe_slow);
 8927 %}
 8928 
 8929 // This pattern is generated automatically from cas.m4.
 8930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8931 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8932   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8933   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8934   ins_cost(VOLATILE_REF_COST);
 8935   effect(KILL cr);
 8936   format %{
 8937     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8938     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8939   %}
 8940   ins_encode %{
 8941     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8942                Assembler::word, /*acquire*/ true, /*release*/ true,
 8943                /*weak*/ true, noreg);
 8944     __ csetw($res$$Register, Assembler::EQ);
 8945   %}
 8946   ins_pipe(pipe_slow);
 8947 %}
 8948 
 8949 // This pattern is generated automatically from cas.m4.
 8950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8951 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8952   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8953   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8954   ins_cost(VOLATILE_REF_COST);
 8955   effect(KILL cr);
 8956   format %{
 8957     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8958     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8959   %}
 8960   ins_encode %{
 8961     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8962                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8963                /*weak*/ true, noreg);
 8964     __ csetw($res$$Register, Assembler::EQ);
 8965   %}
 8966   ins_pipe(pipe_slow);
 8967 %}
 8968 
 8969 // END This section of the file is automatically generated. Do not edit --------------
 8970 // ---------------------------------------------------------------------
 8971 
 8972 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 8973   match(Set prev (GetAndSetI mem newv));
 8974   ins_cost(2 * VOLATILE_REF_COST);
 8975   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 8976   ins_encode %{
 8977     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8978   %}
 8979   ins_pipe(pipe_serial);
 8980 %}
 8981 
 8982 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 8983   match(Set prev (GetAndSetL mem newv));
 8984   ins_cost(2 * VOLATILE_REF_COST);
 8985   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 8986   ins_encode %{
 8987     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8988   %}
 8989   ins_pipe(pipe_serial);
 8990 %}
 8991 
 8992 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 8993   predicate(n->as_LoadStore()->barrier_data() == 0);
 8994   match(Set prev (GetAndSetN mem newv));
 8995   ins_cost(2 * VOLATILE_REF_COST);
 8996   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 8997   ins_encode %{
 8998     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8999   %}
 9000   ins_pipe(pipe_serial);
 9001 %}
 9002 
 9003 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9004   predicate(n->as_LoadStore()->barrier_data() == 0);
 9005   match(Set prev (GetAndSetP mem newv));
 9006   ins_cost(2 * VOLATILE_REF_COST);
 9007   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9008   ins_encode %{
 9009     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9010   %}
 9011   ins_pipe(pipe_serial);
 9012 %}
 9013 
 9014 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9015   predicate(needs_acquiring_load_exclusive(n));
 9016   match(Set prev (GetAndSetI mem newv));
 9017   ins_cost(VOLATILE_REF_COST);
 9018   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9019   ins_encode %{
 9020     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9021   %}
 9022   ins_pipe(pipe_serial);
 9023 %}
 9024 
 9025 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9026   predicate(needs_acquiring_load_exclusive(n));
 9027   match(Set prev (GetAndSetL mem newv));
 9028   ins_cost(VOLATILE_REF_COST);
 9029   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9030   ins_encode %{
 9031     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9032   %}
 9033   ins_pipe(pipe_serial);
 9034 %}
 9035 
 9036 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9037   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9038   match(Set prev (GetAndSetN mem newv));
 9039   ins_cost(VOLATILE_REF_COST);
 9040   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9041   ins_encode %{
 9042     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9043   %}
 9044   ins_pipe(pipe_serial);
 9045 %}
 9046 
 9047 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9048   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9049   match(Set prev (GetAndSetP mem newv));
 9050   ins_cost(VOLATILE_REF_COST);
 9051   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9052   ins_encode %{
 9053     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9054   %}
 9055   ins_pipe(pipe_serial);
 9056 %}
 9057 
 9058 
 9059 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9060   match(Set newval (GetAndAddL mem incr));
 9061   ins_cost(2 * VOLATILE_REF_COST + 1);
 9062   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9063   ins_encode %{
 9064     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9065   %}
 9066   ins_pipe(pipe_serial);
 9067 %}
 9068 
 9069 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9070   predicate(n->as_LoadStore()->result_not_used());
 9071   match(Set dummy (GetAndAddL mem incr));
 9072   ins_cost(2 * VOLATILE_REF_COST);
 9073   format %{ "get_and_addL [$mem], $incr" %}
 9074   ins_encode %{
 9075     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9076   %}
 9077   ins_pipe(pipe_serial);
 9078 %}
 9079 
 9080 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9081   match(Set newval (GetAndAddL mem incr));
 9082   ins_cost(2 * VOLATILE_REF_COST + 1);
 9083   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9084   ins_encode %{
 9085     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9086   %}
 9087   ins_pipe(pipe_serial);
 9088 %}
 9089 
 9090 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9091   predicate(n->as_LoadStore()->result_not_used());
 9092   match(Set dummy (GetAndAddL mem incr));
 9093   ins_cost(2 * VOLATILE_REF_COST);
 9094   format %{ "get_and_addL [$mem], $incr" %}
 9095   ins_encode %{
 9096     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9097   %}
 9098   ins_pipe(pipe_serial);
 9099 %}
 9100 
 9101 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9102   match(Set newval (GetAndAddI mem incr));
 9103   ins_cost(2 * VOLATILE_REF_COST + 1);
 9104   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9105   ins_encode %{
 9106     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9107   %}
 9108   ins_pipe(pipe_serial);
 9109 %}
 9110 
 9111 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9112   predicate(n->as_LoadStore()->result_not_used());
 9113   match(Set dummy (GetAndAddI mem incr));
 9114   ins_cost(2 * VOLATILE_REF_COST);
 9115   format %{ "get_and_addI [$mem], $incr" %}
 9116   ins_encode %{
 9117     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9118   %}
 9119   ins_pipe(pipe_serial);
 9120 %}
 9121 
 9122 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9123   match(Set newval (GetAndAddI mem incr));
 9124   ins_cost(2 * VOLATILE_REF_COST + 1);
 9125   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9126   ins_encode %{
 9127     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9128   %}
 9129   ins_pipe(pipe_serial);
 9130 %}
 9131 
 9132 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9133   predicate(n->as_LoadStore()->result_not_used());
 9134   match(Set dummy (GetAndAddI mem incr));
 9135   ins_cost(2 * VOLATILE_REF_COST);
 9136   format %{ "get_and_addI [$mem], $incr" %}
 9137   ins_encode %{
 9138     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9139   %}
 9140   ins_pipe(pipe_serial);
 9141 %}
 9142 
 9143 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9144   predicate(needs_acquiring_load_exclusive(n));
 9145   match(Set newval (GetAndAddL mem incr));
 9146   ins_cost(VOLATILE_REF_COST + 1);
 9147   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9148   ins_encode %{
 9149     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9150   %}
 9151   ins_pipe(pipe_serial);
 9152 %}
 9153 
 9154 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9155   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9156   match(Set dummy (GetAndAddL mem incr));
 9157   ins_cost(VOLATILE_REF_COST);
 9158   format %{ "get_and_addL_acq [$mem], $incr" %}
 9159   ins_encode %{
 9160     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9161   %}
 9162   ins_pipe(pipe_serial);
 9163 %}
 9164 
 9165 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9166   predicate(needs_acquiring_load_exclusive(n));
 9167   match(Set newval (GetAndAddL mem incr));
 9168   ins_cost(VOLATILE_REF_COST + 1);
 9169   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9170   ins_encode %{
 9171     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9172   %}
 9173   ins_pipe(pipe_serial);
 9174 %}
 9175 
 9176 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9177   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9178   match(Set dummy (GetAndAddL mem incr));
 9179   ins_cost(VOLATILE_REF_COST);
 9180   format %{ "get_and_addL_acq [$mem], $incr" %}
 9181   ins_encode %{
 9182     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9183   %}
 9184   ins_pipe(pipe_serial);
 9185 %}
 9186 
 9187 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9188   predicate(needs_acquiring_load_exclusive(n));
 9189   match(Set newval (GetAndAddI mem incr));
 9190   ins_cost(VOLATILE_REF_COST + 1);
 9191   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9192   ins_encode %{
 9193     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9194   %}
 9195   ins_pipe(pipe_serial);
 9196 %}
 9197 
 9198 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9199   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9200   match(Set dummy (GetAndAddI mem incr));
 9201   ins_cost(VOLATILE_REF_COST);
 9202   format %{ "get_and_addI_acq [$mem], $incr" %}
 9203   ins_encode %{
 9204     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9205   %}
 9206   ins_pipe(pipe_serial);
 9207 %}
 9208 
 9209 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9210   predicate(needs_acquiring_load_exclusive(n));
 9211   match(Set newval (GetAndAddI mem incr));
 9212   ins_cost(VOLATILE_REF_COST + 1);
 9213   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9214   ins_encode %{
 9215     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9216   %}
 9217   ins_pipe(pipe_serial);
 9218 %}
 9219 
 9220 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9221   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9222   match(Set dummy (GetAndAddI mem incr));
 9223   ins_cost(VOLATILE_REF_COST);
 9224   format %{ "get_and_addI_acq [$mem], $incr" %}
 9225   ins_encode %{
 9226     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9227   %}
 9228   ins_pipe(pipe_serial);
 9229 %}
 9230 
 9231 // Manifest a CmpU result in an integer register.
 9232 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9233 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9234 %{
 9235   match(Set dst (CmpU3 src1 src2));
 9236   effect(KILL flags);
 9237 
 9238   ins_cost(INSN_COST * 3);
 9239   format %{
 9240       "cmpw $src1, $src2\n\t"
 9241       "csetw $dst, ne\n\t"
 9242       "cnegw $dst, lo\t# CmpU3(reg)"
 9243   %}
 9244   ins_encode %{
 9245     __ cmpw($src1$$Register, $src2$$Register);
 9246     __ csetw($dst$$Register, Assembler::NE);
 9247     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9248   %}
 9249 
 9250   ins_pipe(pipe_class_default);
 9251 %}
 9252 
 9253 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9254 %{
 9255   match(Set dst (CmpU3 src1 src2));
 9256   effect(KILL flags);
 9257 
 9258   ins_cost(INSN_COST * 3);
 9259   format %{
 9260       "subsw zr, $src1, $src2\n\t"
 9261       "csetw $dst, ne\n\t"
 9262       "cnegw $dst, lo\t# CmpU3(imm)"
 9263   %}
 9264   ins_encode %{
 9265     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9266     __ csetw($dst$$Register, Assembler::NE);
 9267     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9268   %}
 9269 
 9270   ins_pipe(pipe_class_default);
 9271 %}
 9272 
 9273 // Manifest a CmpUL result in an integer register.
 9274 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9275 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9276 %{
 9277   match(Set dst (CmpUL3 src1 src2));
 9278   effect(KILL flags);
 9279 
 9280   ins_cost(INSN_COST * 3);
 9281   format %{
 9282       "cmp $src1, $src2\n\t"
 9283       "csetw $dst, ne\n\t"
 9284       "cnegw $dst, lo\t# CmpUL3(reg)"
 9285   %}
 9286   ins_encode %{
 9287     __ cmp($src1$$Register, $src2$$Register);
 9288     __ csetw($dst$$Register, Assembler::NE);
 9289     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9290   %}
 9291 
 9292   ins_pipe(pipe_class_default);
 9293 %}
 9294 
 9295 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9296 %{
 9297   match(Set dst (CmpUL3 src1 src2));
 9298   effect(KILL flags);
 9299 
 9300   ins_cost(INSN_COST * 3);
 9301   format %{
 9302       "subs zr, $src1, $src2\n\t"
 9303       "csetw $dst, ne\n\t"
 9304       "cnegw $dst, lo\t# CmpUL3(imm)"
 9305   %}
 9306   ins_encode %{
 9307     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9308     __ csetw($dst$$Register, Assembler::NE);
 9309     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9310   %}
 9311 
 9312   ins_pipe(pipe_class_default);
 9313 %}
 9314 
 9315 // Manifest a CmpL result in an integer register.
 9316 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9317 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9318 %{
 9319   match(Set dst (CmpL3 src1 src2));
 9320   effect(KILL flags);
 9321 
 9322   ins_cost(INSN_COST * 3);
 9323   format %{
 9324       "cmp $src1, $src2\n\t"
 9325       "csetw $dst, ne\n\t"
 9326       "cnegw $dst, lt\t# CmpL3(reg)"
 9327   %}
 9328   ins_encode %{
 9329     __ cmp($src1$$Register, $src2$$Register);
 9330     __ csetw($dst$$Register, Assembler::NE);
 9331     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9332   %}
 9333 
 9334   ins_pipe(pipe_class_default);
 9335 %}
 9336 
 9337 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9338 %{
 9339   match(Set dst (CmpL3 src1 src2));
 9340   effect(KILL flags);
 9341 
 9342   ins_cost(INSN_COST * 3);
 9343   format %{
 9344       "subs zr, $src1, $src2\n\t"
 9345       "csetw $dst, ne\n\t"
 9346       "cnegw $dst, lt\t# CmpL3(imm)"
 9347   %}
 9348   ins_encode %{
 9349     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9350     __ csetw($dst$$Register, Assembler::NE);
 9351     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9352   %}
 9353 
 9354   ins_pipe(pipe_class_default);
 9355 %}
 9356 
 9357 // ============================================================================
 9358 // Conditional Move Instructions
 9359 
 9360 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9361 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9362 // define an op class which merged both inputs and use it to type the
 9363 // argument to a single rule. unfortunatelyt his fails because the
 9364 // opclass does not live up to the COND_INTER interface of its
 9365 // component operands. When the generic code tries to negate the
 9366 // operand it ends up running the generci Machoper::negate method
 9367 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9368 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9369 
 9370 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9371   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9372 
 9373   ins_cost(INSN_COST * 2);
 9374   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9375 
 9376   ins_encode %{
 9377     __ cselw(as_Register($dst$$reg),
 9378              as_Register($src2$$reg),
 9379              as_Register($src1$$reg),
 9380              (Assembler::Condition)$cmp$$cmpcode);
 9381   %}
 9382 
 9383   ins_pipe(icond_reg_reg);
 9384 %}
 9385 
 9386 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9387   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9388 
 9389   ins_cost(INSN_COST * 2);
 9390   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9391 
 9392   ins_encode %{
 9393     __ cselw(as_Register($dst$$reg),
 9394              as_Register($src2$$reg),
 9395              as_Register($src1$$reg),
 9396              (Assembler::Condition)$cmp$$cmpcode);
 9397   %}
 9398 
 9399   ins_pipe(icond_reg_reg);
 9400 %}
 9401 
 9402 // special cases where one arg is zero
 9403 
 9404 // n.b. this is selected in preference to the rule above because it
 9405 // avoids loading constant 0 into a source register
 9406 
 9407 // TODO
 9408 // we ought only to be able to cull one of these variants as the ideal
 9409 // transforms ought always to order the zero consistently (to left/right?)
 9410 
 9411 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9412   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9413 
 9414   ins_cost(INSN_COST * 2);
 9415   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9416 
 9417   ins_encode %{
 9418     __ cselw(as_Register($dst$$reg),
 9419              as_Register($src$$reg),
 9420              zr,
 9421              (Assembler::Condition)$cmp$$cmpcode);
 9422   %}
 9423 
 9424   ins_pipe(icond_reg);
 9425 %}
 9426 
 9427 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9428   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9429 
 9430   ins_cost(INSN_COST * 2);
 9431   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9432 
 9433   ins_encode %{
 9434     __ cselw(as_Register($dst$$reg),
 9435              as_Register($src$$reg),
 9436              zr,
 9437              (Assembler::Condition)$cmp$$cmpcode);
 9438   %}
 9439 
 9440   ins_pipe(icond_reg);
 9441 %}
 9442 
 9443 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9444   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9445 
 9446   ins_cost(INSN_COST * 2);
 9447   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9448 
 9449   ins_encode %{
 9450     __ cselw(as_Register($dst$$reg),
 9451              zr,
 9452              as_Register($src$$reg),
 9453              (Assembler::Condition)$cmp$$cmpcode);
 9454   %}
 9455 
 9456   ins_pipe(icond_reg);
 9457 %}
 9458 
 9459 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9460   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9461 
 9462   ins_cost(INSN_COST * 2);
 9463   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9464 
 9465   ins_encode %{
 9466     __ cselw(as_Register($dst$$reg),
 9467              zr,
 9468              as_Register($src$$reg),
 9469              (Assembler::Condition)$cmp$$cmpcode);
 9470   %}
 9471 
 9472   ins_pipe(icond_reg);
 9473 %}
 9474 
 9475 // special case for creating a boolean 0 or 1
 9476 
 9477 // n.b. this is selected in preference to the rule above because it
 9478 // avoids loading constants 0 and 1 into a source register
 9479 
 9480 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9481   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9482 
 9483   ins_cost(INSN_COST * 2);
 9484   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9485 
 9486   ins_encode %{
 9487     // equivalently
 9488     // cset(as_Register($dst$$reg),
 9489     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9490     __ csincw(as_Register($dst$$reg),
 9491              zr,
 9492              zr,
 9493              (Assembler::Condition)$cmp$$cmpcode);
 9494   %}
 9495 
 9496   ins_pipe(icond_none);
 9497 %}
 9498 
 9499 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9500   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9501 
 9502   ins_cost(INSN_COST * 2);
 9503   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9504 
 9505   ins_encode %{
 9506     // equivalently
 9507     // cset(as_Register($dst$$reg),
 9508     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9509     __ csincw(as_Register($dst$$reg),
 9510              zr,
 9511              zr,
 9512              (Assembler::Condition)$cmp$$cmpcode);
 9513   %}
 9514 
 9515   ins_pipe(icond_none);
 9516 %}
 9517 
 9518 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9519   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9520 
 9521   ins_cost(INSN_COST * 2);
 9522   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9523 
 9524   ins_encode %{
 9525     __ csel(as_Register($dst$$reg),
 9526             as_Register($src2$$reg),
 9527             as_Register($src1$$reg),
 9528             (Assembler::Condition)$cmp$$cmpcode);
 9529   %}
 9530 
 9531   ins_pipe(icond_reg_reg);
 9532 %}
 9533 
 9534 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9535   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9536 
 9537   ins_cost(INSN_COST * 2);
 9538   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9539 
 9540   ins_encode %{
 9541     __ csel(as_Register($dst$$reg),
 9542             as_Register($src2$$reg),
 9543             as_Register($src1$$reg),
 9544             (Assembler::Condition)$cmp$$cmpcode);
 9545   %}
 9546 
 9547   ins_pipe(icond_reg_reg);
 9548 %}
 9549 
 9550 // special cases where one arg is zero
 9551 
 9552 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9553   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9554 
 9555   ins_cost(INSN_COST * 2);
 9556   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9557 
 9558   ins_encode %{
 9559     __ csel(as_Register($dst$$reg),
 9560             zr,
 9561             as_Register($src$$reg),
 9562             (Assembler::Condition)$cmp$$cmpcode);
 9563   %}
 9564 
 9565   ins_pipe(icond_reg);
 9566 %}
 9567 
 9568 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9569   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9570 
 9571   ins_cost(INSN_COST * 2);
 9572   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9573 
 9574   ins_encode %{
 9575     __ csel(as_Register($dst$$reg),
 9576             zr,
 9577             as_Register($src$$reg),
 9578             (Assembler::Condition)$cmp$$cmpcode);
 9579   %}
 9580 
 9581   ins_pipe(icond_reg);
 9582 %}
 9583 
 9584 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9585   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9586 
 9587   ins_cost(INSN_COST * 2);
 9588   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9589 
 9590   ins_encode %{
 9591     __ csel(as_Register($dst$$reg),
 9592             as_Register($src$$reg),
 9593             zr,
 9594             (Assembler::Condition)$cmp$$cmpcode);
 9595   %}
 9596 
 9597   ins_pipe(icond_reg);
 9598 %}
 9599 
 9600 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9601   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9602 
 9603   ins_cost(INSN_COST * 2);
 9604   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9605 
 9606   ins_encode %{
 9607     __ csel(as_Register($dst$$reg),
 9608             as_Register($src$$reg),
 9609             zr,
 9610             (Assembler::Condition)$cmp$$cmpcode);
 9611   %}
 9612 
 9613   ins_pipe(icond_reg);
 9614 %}
 9615 
 9616 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9617   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9618 
 9619   ins_cost(INSN_COST * 2);
 9620   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9621 
 9622   ins_encode %{
 9623     __ csel(as_Register($dst$$reg),
 9624             as_Register($src2$$reg),
 9625             as_Register($src1$$reg),
 9626             (Assembler::Condition)$cmp$$cmpcode);
 9627   %}
 9628 
 9629   ins_pipe(icond_reg_reg);
 9630 %}
 9631 
 9632 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9633   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9634 
 9635   ins_cost(INSN_COST * 2);
 9636   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9637 
 9638   ins_encode %{
 9639     __ csel(as_Register($dst$$reg),
 9640             as_Register($src2$$reg),
 9641             as_Register($src1$$reg),
 9642             (Assembler::Condition)$cmp$$cmpcode);
 9643   %}
 9644 
 9645   ins_pipe(icond_reg_reg);
 9646 %}
 9647 
 9648 // special cases where one arg is zero
 9649 
 9650 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9651   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9652 
 9653   ins_cost(INSN_COST * 2);
 9654   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9655 
 9656   ins_encode %{
 9657     __ csel(as_Register($dst$$reg),
 9658             zr,
 9659             as_Register($src$$reg),
 9660             (Assembler::Condition)$cmp$$cmpcode);
 9661   %}
 9662 
 9663   ins_pipe(icond_reg);
 9664 %}
 9665 
 9666 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9667   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9668 
 9669   ins_cost(INSN_COST * 2);
 9670   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9671 
 9672   ins_encode %{
 9673     __ csel(as_Register($dst$$reg),
 9674             zr,
 9675             as_Register($src$$reg),
 9676             (Assembler::Condition)$cmp$$cmpcode);
 9677   %}
 9678 
 9679   ins_pipe(icond_reg);
 9680 %}
 9681 
 9682 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9683   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9684 
 9685   ins_cost(INSN_COST * 2);
 9686   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9687 
 9688   ins_encode %{
 9689     __ csel(as_Register($dst$$reg),
 9690             as_Register($src$$reg),
 9691             zr,
 9692             (Assembler::Condition)$cmp$$cmpcode);
 9693   %}
 9694 
 9695   ins_pipe(icond_reg);
 9696 %}
 9697 
 9698 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9699   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9700 
 9701   ins_cost(INSN_COST * 2);
 9702   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9703 
 9704   ins_encode %{
 9705     __ csel(as_Register($dst$$reg),
 9706             as_Register($src$$reg),
 9707             zr,
 9708             (Assembler::Condition)$cmp$$cmpcode);
 9709   %}
 9710 
 9711   ins_pipe(icond_reg);
 9712 %}
 9713 
 9714 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9715   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9716 
 9717   ins_cost(INSN_COST * 2);
 9718   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9719 
 9720   ins_encode %{
 9721     __ cselw(as_Register($dst$$reg),
 9722              as_Register($src2$$reg),
 9723              as_Register($src1$$reg),
 9724              (Assembler::Condition)$cmp$$cmpcode);
 9725   %}
 9726 
 9727   ins_pipe(icond_reg_reg);
 9728 %}
 9729 
 9730 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9731   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9732 
 9733   ins_cost(INSN_COST * 2);
 9734   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9735 
 9736   ins_encode %{
 9737     __ cselw(as_Register($dst$$reg),
 9738              as_Register($src2$$reg),
 9739              as_Register($src1$$reg),
 9740              (Assembler::Condition)$cmp$$cmpcode);
 9741   %}
 9742 
 9743   ins_pipe(icond_reg_reg);
 9744 %}
 9745 
 9746 // special cases where one arg is zero
 9747 
 9748 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9749   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9750 
 9751   ins_cost(INSN_COST * 2);
 9752   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9753 
 9754   ins_encode %{
 9755     __ cselw(as_Register($dst$$reg),
 9756              zr,
 9757              as_Register($src$$reg),
 9758              (Assembler::Condition)$cmp$$cmpcode);
 9759   %}
 9760 
 9761   ins_pipe(icond_reg);
 9762 %}
 9763 
 9764 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9765   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9766 
 9767   ins_cost(INSN_COST * 2);
 9768   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9769 
 9770   ins_encode %{
 9771     __ cselw(as_Register($dst$$reg),
 9772              zr,
 9773              as_Register($src$$reg),
 9774              (Assembler::Condition)$cmp$$cmpcode);
 9775   %}
 9776 
 9777   ins_pipe(icond_reg);
 9778 %}
 9779 
 9780 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9781   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9782 
 9783   ins_cost(INSN_COST * 2);
 9784   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9785 
 9786   ins_encode %{
 9787     __ cselw(as_Register($dst$$reg),
 9788              as_Register($src$$reg),
 9789              zr,
 9790              (Assembler::Condition)$cmp$$cmpcode);
 9791   %}
 9792 
 9793   ins_pipe(icond_reg);
 9794 %}
 9795 
 9796 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9797   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9798 
 9799   ins_cost(INSN_COST * 2);
 9800   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9801 
 9802   ins_encode %{
 9803     __ cselw(as_Register($dst$$reg),
 9804              as_Register($src$$reg),
 9805              zr,
 9806              (Assembler::Condition)$cmp$$cmpcode);
 9807   %}
 9808 
 9809   ins_pipe(icond_reg);
 9810 %}
 9811 
 9812 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9813 %{
 9814   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9815 
 9816   ins_cost(INSN_COST * 3);
 9817 
 9818   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9819   ins_encode %{
 9820     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9821     __ fcsels(as_FloatRegister($dst$$reg),
 9822               as_FloatRegister($src2$$reg),
 9823               as_FloatRegister($src1$$reg),
 9824               cond);
 9825   %}
 9826 
 9827   ins_pipe(fp_cond_reg_reg_s);
 9828 %}
 9829 
 9830 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
 9831 %{
 9832   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9833 
 9834   ins_cost(INSN_COST * 3);
 9835 
 9836   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9837   ins_encode %{
 9838     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9839     __ fcsels(as_FloatRegister($dst$$reg),
 9840               as_FloatRegister($src2$$reg),
 9841               as_FloatRegister($src1$$reg),
 9842               cond);
 9843   %}
 9844 
 9845   ins_pipe(fp_cond_reg_reg_s);
 9846 %}
 9847 
 9848 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
 9849 %{
 9850   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9851 
 9852   ins_cost(INSN_COST * 3);
 9853 
 9854   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9855   ins_encode %{
 9856     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9857     __ fcseld(as_FloatRegister($dst$$reg),
 9858               as_FloatRegister($src2$$reg),
 9859               as_FloatRegister($src1$$reg),
 9860               cond);
 9861   %}
 9862 
 9863   ins_pipe(fp_cond_reg_reg_d);
 9864 %}
 9865 
 9866 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
 9867 %{
 9868   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9869 
 9870   ins_cost(INSN_COST * 3);
 9871 
 9872   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9873   ins_encode %{
 9874     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9875     __ fcseld(as_FloatRegister($dst$$reg),
 9876               as_FloatRegister($src2$$reg),
 9877               as_FloatRegister($src1$$reg),
 9878               cond);
 9879   %}
 9880 
 9881   ins_pipe(fp_cond_reg_reg_d);
 9882 %}
 9883 
 9884 // ============================================================================
 9885 // Arithmetic Instructions
 9886 //
 9887 
 9888 // Integer Addition
 9889 
 9890 // TODO
 9891 // these currently employ operations which do not set CR and hence are
 9892 // not flagged as killing CR but we would like to isolate the cases
 9893 // where we want to set flags from those where we don't. need to work
 9894 // out how to do that.
 9895 
 9896 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9897   match(Set dst (AddI src1 src2));
 9898 
 9899   ins_cost(INSN_COST);
 9900   format %{ "addw  $dst, $src1, $src2" %}
 9901 
 9902   ins_encode %{
 9903     __ addw(as_Register($dst$$reg),
 9904             as_Register($src1$$reg),
 9905             as_Register($src2$$reg));
 9906   %}
 9907 
 9908   ins_pipe(ialu_reg_reg);
 9909 %}
 9910 
 9911 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9912   match(Set dst (AddI src1 src2));
 9913 
 9914   ins_cost(INSN_COST);
 9915   format %{ "addw $dst, $src1, $src2" %}
 9916 
 9917   // use opcode to indicate that this is an add not a sub
 9918   opcode(0x0);
 9919 
 9920   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9921 
 9922   ins_pipe(ialu_reg_imm);
 9923 %}
 9924 
 9925 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
 9926   match(Set dst (AddI (ConvL2I src1) src2));
 9927 
 9928   ins_cost(INSN_COST);
 9929   format %{ "addw $dst, $src1, $src2" %}
 9930 
 9931   // use opcode to indicate that this is an add not a sub
 9932   opcode(0x0);
 9933 
 9934   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9935 
 9936   ins_pipe(ialu_reg_imm);
 9937 %}
 9938 
 9939 // Pointer Addition
 9940 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
 9941   match(Set dst (AddP src1 src2));
 9942 
 9943   ins_cost(INSN_COST);
 9944   format %{ "add $dst, $src1, $src2\t# ptr" %}
 9945 
 9946   ins_encode %{
 9947     __ add(as_Register($dst$$reg),
 9948            as_Register($src1$$reg),
 9949            as_Register($src2$$reg));
 9950   %}
 9951 
 9952   ins_pipe(ialu_reg_reg);
 9953 %}
 9954 
 9955 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
 9956   match(Set dst (AddP src1 (ConvI2L src2)));
 9957 
 9958   ins_cost(1.9 * INSN_COST);
 9959   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
 9960 
 9961   ins_encode %{
 9962     __ add(as_Register($dst$$reg),
 9963            as_Register($src1$$reg),
 9964            as_Register($src2$$reg), ext::sxtw);
 9965   %}
 9966 
 9967   ins_pipe(ialu_reg_reg);
 9968 %}
 9969 
 9970 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
 9971   match(Set dst (AddP src1 (LShiftL src2 scale)));
 9972 
 9973   ins_cost(1.9 * INSN_COST);
 9974   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
 9975 
 9976   ins_encode %{
 9977     __ lea(as_Register($dst$$reg),
 9978            Address(as_Register($src1$$reg), as_Register($src2$$reg),
 9979                    Address::lsl($scale$$constant)));
 9980   %}
 9981 
 9982   ins_pipe(ialu_reg_reg_shift);
 9983 %}
 9984 
 9985 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
 9986   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
 9987 
 9988   ins_cost(1.9 * INSN_COST);
 9989   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
 9990 
 9991   ins_encode %{
 9992     __ lea(as_Register($dst$$reg),
 9993            Address(as_Register($src1$$reg), as_Register($src2$$reg),
 9994                    Address::sxtw($scale$$constant)));
 9995   %}
 9996 
 9997   ins_pipe(ialu_reg_reg_shift);
 9998 %}
 9999 
10000 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10001   match(Set dst (LShiftL (ConvI2L src) scale));
10002 
10003   ins_cost(INSN_COST);
10004   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10005 
10006   ins_encode %{
10007     __ sbfiz(as_Register($dst$$reg),
10008           as_Register($src$$reg),
10009           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10010   %}
10011 
10012   ins_pipe(ialu_reg_shift);
10013 %}
10014 
10015 // Pointer Immediate Addition
10016 // n.b. this needs to be more expensive than using an indirect memory
10017 // operand
10018 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10019   match(Set dst (AddP src1 src2));
10020 
10021   ins_cost(INSN_COST);
10022   format %{ "add $dst, $src1, $src2\t# ptr" %}
10023 
10024   // use opcode to indicate that this is an add not a sub
10025   opcode(0x0);
10026 
10027   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10028 
10029   ins_pipe(ialu_reg_imm);
10030 %}
10031 
10032 // Long Addition
10033 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10034 
10035   match(Set dst (AddL src1 src2));
10036 
10037   ins_cost(INSN_COST);
10038   format %{ "add  $dst, $src1, $src2" %}
10039 
10040   ins_encode %{
10041     __ add(as_Register($dst$$reg),
10042            as_Register($src1$$reg),
10043            as_Register($src2$$reg));
10044   %}
10045 
10046   ins_pipe(ialu_reg_reg);
10047 %}
10048 
10049 // No constant pool entries requiredLong Immediate Addition.
10050 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10051   match(Set dst (AddL src1 src2));
10052 
10053   ins_cost(INSN_COST);
10054   format %{ "add $dst, $src1, $src2" %}
10055 
10056   // use opcode to indicate that this is an add not a sub
10057   opcode(0x0);
10058 
10059   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10060 
10061   ins_pipe(ialu_reg_imm);
10062 %}
10063 
10064 // Integer Subtraction
10065 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10066   match(Set dst (SubI src1 src2));
10067 
10068   ins_cost(INSN_COST);
10069   format %{ "subw  $dst, $src1, $src2" %}
10070 
10071   ins_encode %{
10072     __ subw(as_Register($dst$$reg),
10073             as_Register($src1$$reg),
10074             as_Register($src2$$reg));
10075   %}
10076 
10077   ins_pipe(ialu_reg_reg);
10078 %}
10079 
10080 // Immediate Subtraction
10081 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10082   match(Set dst (SubI src1 src2));
10083 
10084   ins_cost(INSN_COST);
10085   format %{ "subw $dst, $src1, $src2" %}
10086 
10087   // use opcode to indicate that this is a sub not an add
10088   opcode(0x1);
10089 
10090   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10091 
10092   ins_pipe(ialu_reg_imm);
10093 %}
10094 
10095 // Long Subtraction
10096 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10097 
10098   match(Set dst (SubL src1 src2));
10099 
10100   ins_cost(INSN_COST);
10101   format %{ "sub  $dst, $src1, $src2" %}
10102 
10103   ins_encode %{
10104     __ sub(as_Register($dst$$reg),
10105            as_Register($src1$$reg),
10106            as_Register($src2$$reg));
10107   %}
10108 
10109   ins_pipe(ialu_reg_reg);
10110 %}
10111 
10112 // No constant pool entries requiredLong Immediate Subtraction.
10113 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10114   match(Set dst (SubL src1 src2));
10115 
10116   ins_cost(INSN_COST);
10117   format %{ "sub$dst, $src1, $src2" %}
10118 
10119   // use opcode to indicate that this is a sub not an add
10120   opcode(0x1);
10121 
10122   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10123 
10124   ins_pipe(ialu_reg_imm);
10125 %}
10126 
10127 // Integer Negation (special case for sub)
10128 
10129 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10130   match(Set dst (SubI zero src));
10131 
10132   ins_cost(INSN_COST);
10133   format %{ "negw $dst, $src\t# int" %}
10134 
10135   ins_encode %{
10136     __ negw(as_Register($dst$$reg),
10137             as_Register($src$$reg));
10138   %}
10139 
10140   ins_pipe(ialu_reg);
10141 %}
10142 
10143 // Long Negation
10144 
10145 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10146   match(Set dst (SubL zero src));
10147 
10148   ins_cost(INSN_COST);
10149   format %{ "neg $dst, $src\t# long" %}
10150 
10151   ins_encode %{
10152     __ neg(as_Register($dst$$reg),
10153            as_Register($src$$reg));
10154   %}
10155 
10156   ins_pipe(ialu_reg);
10157 %}
10158 
10159 // Integer Multiply
10160 
10161 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10162   match(Set dst (MulI src1 src2));
10163 
10164   ins_cost(INSN_COST * 3);
10165   format %{ "mulw  $dst, $src1, $src2" %}
10166 
10167   ins_encode %{
10168     __ mulw(as_Register($dst$$reg),
10169             as_Register($src1$$reg),
10170             as_Register($src2$$reg));
10171   %}
10172 
10173   ins_pipe(imul_reg_reg);
10174 %}
10175 
10176 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10177   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10178 
10179   ins_cost(INSN_COST * 3);
10180   format %{ "smull  $dst, $src1, $src2" %}
10181 
10182   ins_encode %{
10183     __ smull(as_Register($dst$$reg),
10184              as_Register($src1$$reg),
10185              as_Register($src2$$reg));
10186   %}
10187 
10188   ins_pipe(imul_reg_reg);
10189 %}
10190 
10191 // Long Multiply
10192 
10193 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10194   match(Set dst (MulL src1 src2));
10195 
10196   ins_cost(INSN_COST * 5);
10197   format %{ "mul  $dst, $src1, $src2" %}
10198 
10199   ins_encode %{
10200     __ mul(as_Register($dst$$reg),
10201            as_Register($src1$$reg),
10202            as_Register($src2$$reg));
10203   %}
10204 
10205   ins_pipe(lmul_reg_reg);
10206 %}
10207 
10208 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10209 %{
10210   match(Set dst (MulHiL src1 src2));
10211 
10212   ins_cost(INSN_COST * 7);
10213   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10214 
10215   ins_encode %{
10216     __ smulh(as_Register($dst$$reg),
10217              as_Register($src1$$reg),
10218              as_Register($src2$$reg));
10219   %}
10220 
10221   ins_pipe(lmul_reg_reg);
10222 %}
10223 
10224 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10225 %{
10226   match(Set dst (UMulHiL src1 src2));
10227 
10228   ins_cost(INSN_COST * 7);
10229   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10230 
10231   ins_encode %{
10232     __ umulh(as_Register($dst$$reg),
10233              as_Register($src1$$reg),
10234              as_Register($src2$$reg));
10235   %}
10236 
10237   ins_pipe(lmul_reg_reg);
10238 %}
10239 
10240 // Combined Integer Multiply & Add/Sub
10241 
10242 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10243   match(Set dst (AddI src3 (MulI src1 src2)));
10244 
10245   ins_cost(INSN_COST * 3);
10246   format %{ "madd  $dst, $src1, $src2, $src3" %}
10247 
10248   ins_encode %{
10249     __ maddw(as_Register($dst$$reg),
10250              as_Register($src1$$reg),
10251              as_Register($src2$$reg),
10252              as_Register($src3$$reg));
10253   %}
10254 
10255   ins_pipe(imac_reg_reg);
10256 %}
10257 
10258 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10259   match(Set dst (SubI src3 (MulI src1 src2)));
10260 
10261   ins_cost(INSN_COST * 3);
10262   format %{ "msub  $dst, $src1, $src2, $src3" %}
10263 
10264   ins_encode %{
10265     __ msubw(as_Register($dst$$reg),
10266              as_Register($src1$$reg),
10267              as_Register($src2$$reg),
10268              as_Register($src3$$reg));
10269   %}
10270 
10271   ins_pipe(imac_reg_reg);
10272 %}
10273 
10274 // Combined Integer Multiply & Neg
10275 
10276 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10277   match(Set dst (MulI (SubI zero src1) src2));
10278 
10279   ins_cost(INSN_COST * 3);
10280   format %{ "mneg  $dst, $src1, $src2" %}
10281 
10282   ins_encode %{
10283     __ mnegw(as_Register($dst$$reg),
10284              as_Register($src1$$reg),
10285              as_Register($src2$$reg));
10286   %}
10287 
10288   ins_pipe(imac_reg_reg);
10289 %}
10290 
10291 // Combined Long Multiply & Add/Sub
10292 
10293 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10294   match(Set dst (AddL src3 (MulL src1 src2)));
10295 
10296   ins_cost(INSN_COST * 5);
10297   format %{ "madd  $dst, $src1, $src2, $src3" %}
10298 
10299   ins_encode %{
10300     __ madd(as_Register($dst$$reg),
10301             as_Register($src1$$reg),
10302             as_Register($src2$$reg),
10303             as_Register($src3$$reg));
10304   %}
10305 
10306   ins_pipe(lmac_reg_reg);
10307 %}
10308 
10309 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10310   match(Set dst (SubL src3 (MulL src1 src2)));
10311 
10312   ins_cost(INSN_COST * 5);
10313   format %{ "msub  $dst, $src1, $src2, $src3" %}
10314 
10315   ins_encode %{
10316     __ msub(as_Register($dst$$reg),
10317             as_Register($src1$$reg),
10318             as_Register($src2$$reg),
10319             as_Register($src3$$reg));
10320   %}
10321 
10322   ins_pipe(lmac_reg_reg);
10323 %}
10324 
10325 // Combined Long Multiply & Neg
10326 
10327 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10328   match(Set dst (MulL (SubL zero src1) src2));
10329 
10330   ins_cost(INSN_COST * 5);
10331   format %{ "mneg  $dst, $src1, $src2" %}
10332 
10333   ins_encode %{
10334     __ mneg(as_Register($dst$$reg),
10335             as_Register($src1$$reg),
10336             as_Register($src2$$reg));
10337   %}
10338 
10339   ins_pipe(lmac_reg_reg);
10340 %}
10341 
10342 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10343 
10344 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10345   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10346 
10347   ins_cost(INSN_COST * 3);
10348   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10349 
10350   ins_encode %{
10351     __ smaddl(as_Register($dst$$reg),
10352               as_Register($src1$$reg),
10353               as_Register($src2$$reg),
10354               as_Register($src3$$reg));
10355   %}
10356 
10357   ins_pipe(imac_reg_reg);
10358 %}
10359 
10360 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10361   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10362 
10363   ins_cost(INSN_COST * 3);
10364   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10365 
10366   ins_encode %{
10367     __ smsubl(as_Register($dst$$reg),
10368               as_Register($src1$$reg),
10369               as_Register($src2$$reg),
10370               as_Register($src3$$reg));
10371   %}
10372 
10373   ins_pipe(imac_reg_reg);
10374 %}
10375 
10376 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10377   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10378 
10379   ins_cost(INSN_COST * 3);
10380   format %{ "smnegl  $dst, $src1, $src2" %}
10381 
10382   ins_encode %{
10383     __ smnegl(as_Register($dst$$reg),
10384               as_Register($src1$$reg),
10385               as_Register($src2$$reg));
10386   %}
10387 
10388   ins_pipe(imac_reg_reg);
10389 %}
10390 
10391 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10392 
10393 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10394   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10395 
10396   ins_cost(INSN_COST * 5);
10397   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10398             "maddw $dst, $src3, $src4, rscratch1" %}
10399 
10400   ins_encode %{
10401     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10402     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10403 
10404   ins_pipe(imac_reg_reg);
10405 %}
10406 
10407 // Integer Divide
10408 
10409 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10410   match(Set dst (DivI src1 src2));
10411 
10412   ins_cost(INSN_COST * 19);
10413   format %{ "sdivw  $dst, $src1, $src2" %}
10414 
10415   ins_encode(aarch64_enc_divw(dst, src1, src2));
10416   ins_pipe(idiv_reg_reg);
10417 %}
10418 
10419 // Long Divide
10420 
10421 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10422   match(Set dst (DivL src1 src2));
10423 
10424   ins_cost(INSN_COST * 35);
10425   format %{ "sdiv   $dst, $src1, $src2" %}
10426 
10427   ins_encode(aarch64_enc_div(dst, src1, src2));
10428   ins_pipe(ldiv_reg_reg);
10429 %}
10430 
10431 // Integer Remainder
10432 
10433 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10434   match(Set dst (ModI src1 src2));
10435 
10436   ins_cost(INSN_COST * 22);
10437   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10438             "msubw  $dst, rscratch1, $src2, $src1" %}
10439 
10440   ins_encode(aarch64_enc_modw(dst, src1, src2));
10441   ins_pipe(idiv_reg_reg);
10442 %}
10443 
10444 // Long Remainder
10445 
10446 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10447   match(Set dst (ModL src1 src2));
10448 
10449   ins_cost(INSN_COST * 38);
10450   format %{ "sdiv   rscratch1, $src1, $src2\n"
10451             "msub   $dst, rscratch1, $src2, $src1" %}
10452 
10453   ins_encode(aarch64_enc_mod(dst, src1, src2));
10454   ins_pipe(ldiv_reg_reg);
10455 %}
10456 
10457 // Unsigned Integer Divide
10458 
10459 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10460   match(Set dst (UDivI src1 src2));
10461 
10462   ins_cost(INSN_COST * 19);
10463   format %{ "udivw  $dst, $src1, $src2" %}
10464 
10465   ins_encode %{
10466     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10467   %}
10468 
10469   ins_pipe(idiv_reg_reg);
10470 %}
10471 
10472 //  Unsigned Long Divide
10473 
10474 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10475   match(Set dst (UDivL src1 src2));
10476 
10477   ins_cost(INSN_COST * 35);
10478   format %{ "udiv   $dst, $src1, $src2" %}
10479 
10480   ins_encode %{
10481     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10482   %}
10483 
10484   ins_pipe(ldiv_reg_reg);
10485 %}
10486 
10487 // Unsigned Integer Remainder
10488 
10489 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10490   match(Set dst (UModI src1 src2));
10491 
10492   ins_cost(INSN_COST * 22);
10493   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10494             "msubw  $dst, rscratch1, $src2, $src1" %}
10495 
10496   ins_encode %{
10497     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10498     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10499   %}
10500 
10501   ins_pipe(idiv_reg_reg);
10502 %}
10503 
10504 // Unsigned Long Remainder
10505 
10506 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10507   match(Set dst (UModL src1 src2));
10508 
10509   ins_cost(INSN_COST * 38);
10510   format %{ "udiv   rscratch1, $src1, $src2\n"
10511             "msub   $dst, rscratch1, $src2, $src1" %}
10512 
10513   ins_encode %{
10514     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10515     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10516   %}
10517 
10518   ins_pipe(ldiv_reg_reg);
10519 %}
10520 
10521 // Integer Shifts
10522 
10523 // Shift Left Register
10524 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10525   match(Set dst (LShiftI src1 src2));
10526 
10527   ins_cost(INSN_COST * 2);
10528   format %{ "lslvw  $dst, $src1, $src2" %}
10529 
10530   ins_encode %{
10531     __ lslvw(as_Register($dst$$reg),
10532              as_Register($src1$$reg),
10533              as_Register($src2$$reg));
10534   %}
10535 
10536   ins_pipe(ialu_reg_reg_vshift);
10537 %}
10538 
10539 // Shift Left Immediate
10540 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10541   match(Set dst (LShiftI src1 src2));
10542 
10543   ins_cost(INSN_COST);
10544   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10545 
10546   ins_encode %{
10547     __ lslw(as_Register($dst$$reg),
10548             as_Register($src1$$reg),
10549             $src2$$constant & 0x1f);
10550   %}
10551 
10552   ins_pipe(ialu_reg_shift);
10553 %}
10554 
10555 // Shift Right Logical Register
10556 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10557   match(Set dst (URShiftI src1 src2));
10558 
10559   ins_cost(INSN_COST * 2);
10560   format %{ "lsrvw  $dst, $src1, $src2" %}
10561 
10562   ins_encode %{
10563     __ lsrvw(as_Register($dst$$reg),
10564              as_Register($src1$$reg),
10565              as_Register($src2$$reg));
10566   %}
10567 
10568   ins_pipe(ialu_reg_reg_vshift);
10569 %}
10570 
10571 // Shift Right Logical Immediate
10572 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10573   match(Set dst (URShiftI src1 src2));
10574 
10575   ins_cost(INSN_COST);
10576   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10577 
10578   ins_encode %{
10579     __ lsrw(as_Register($dst$$reg),
10580             as_Register($src1$$reg),
10581             $src2$$constant & 0x1f);
10582   %}
10583 
10584   ins_pipe(ialu_reg_shift);
10585 %}
10586 
10587 // Shift Right Arithmetic Register
10588 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10589   match(Set dst (RShiftI src1 src2));
10590 
10591   ins_cost(INSN_COST * 2);
10592   format %{ "asrvw  $dst, $src1, $src2" %}
10593 
10594   ins_encode %{
10595     __ asrvw(as_Register($dst$$reg),
10596              as_Register($src1$$reg),
10597              as_Register($src2$$reg));
10598   %}
10599 
10600   ins_pipe(ialu_reg_reg_vshift);
10601 %}
10602 
10603 // Shift Right Arithmetic Immediate
10604 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10605   match(Set dst (RShiftI src1 src2));
10606 
10607   ins_cost(INSN_COST);
10608   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10609 
10610   ins_encode %{
10611     __ asrw(as_Register($dst$$reg),
10612             as_Register($src1$$reg),
10613             $src2$$constant & 0x1f);
10614   %}
10615 
10616   ins_pipe(ialu_reg_shift);
10617 %}
10618 
10619 // Combined Int Mask and Right Shift (using UBFM)
10620 // TODO
10621 
10622 // Long Shifts
10623 
10624 // Shift Left Register
10625 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10626   match(Set dst (LShiftL src1 src2));
10627 
10628   ins_cost(INSN_COST * 2);
10629   format %{ "lslv  $dst, $src1, $src2" %}
10630 
10631   ins_encode %{
10632     __ lslv(as_Register($dst$$reg),
10633             as_Register($src1$$reg),
10634             as_Register($src2$$reg));
10635   %}
10636 
10637   ins_pipe(ialu_reg_reg_vshift);
10638 %}
10639 
10640 // Shift Left Immediate
10641 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10642   match(Set dst (LShiftL src1 src2));
10643 
10644   ins_cost(INSN_COST);
10645   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10646 
10647   ins_encode %{
10648     __ lsl(as_Register($dst$$reg),
10649             as_Register($src1$$reg),
10650             $src2$$constant & 0x3f);
10651   %}
10652 
10653   ins_pipe(ialu_reg_shift);
10654 %}
10655 
10656 // Shift Right Logical Register
10657 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10658   match(Set dst (URShiftL src1 src2));
10659 
10660   ins_cost(INSN_COST * 2);
10661   format %{ "lsrv  $dst, $src1, $src2" %}
10662 
10663   ins_encode %{
10664     __ lsrv(as_Register($dst$$reg),
10665             as_Register($src1$$reg),
10666             as_Register($src2$$reg));
10667   %}
10668 
10669   ins_pipe(ialu_reg_reg_vshift);
10670 %}
10671 
10672 // Shift Right Logical Immediate
10673 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10674   match(Set dst (URShiftL src1 src2));
10675 
10676   ins_cost(INSN_COST);
10677   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10678 
10679   ins_encode %{
10680     __ lsr(as_Register($dst$$reg),
10681            as_Register($src1$$reg),
10682            $src2$$constant & 0x3f);
10683   %}
10684 
10685   ins_pipe(ialu_reg_shift);
10686 %}
10687 
10688 // A special-case pattern for card table stores.
10689 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10690   match(Set dst (URShiftL (CastP2X src1) src2));
10691 
10692   ins_cost(INSN_COST);
10693   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10694 
10695   ins_encode %{
10696     __ lsr(as_Register($dst$$reg),
10697            as_Register($src1$$reg),
10698            $src2$$constant & 0x3f);
10699   %}
10700 
10701   ins_pipe(ialu_reg_shift);
10702 %}
10703 
10704 // Shift Right Arithmetic Register
10705 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10706   match(Set dst (RShiftL src1 src2));
10707 
10708   ins_cost(INSN_COST * 2);
10709   format %{ "asrv  $dst, $src1, $src2" %}
10710 
10711   ins_encode %{
10712     __ asrv(as_Register($dst$$reg),
10713             as_Register($src1$$reg),
10714             as_Register($src2$$reg));
10715   %}
10716 
10717   ins_pipe(ialu_reg_reg_vshift);
10718 %}
10719 
10720 // Shift Right Arithmetic Immediate
10721 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10722   match(Set dst (RShiftL src1 src2));
10723 
10724   ins_cost(INSN_COST);
10725   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10726 
10727   ins_encode %{
10728     __ asr(as_Register($dst$$reg),
10729            as_Register($src1$$reg),
10730            $src2$$constant & 0x3f);
10731   %}
10732 
10733   ins_pipe(ialu_reg_shift);
10734 %}
10735 
10736 // BEGIN This section of the file is automatically generated. Do not edit --------------
10737 // This section is generated from aarch64_ad.m4
10738 
10739 // This pattern is automatically generated from aarch64_ad.m4
10740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10741 instruct regL_not_reg(iRegLNoSp dst,
10742                          iRegL src1, immL_M1 m1,
10743                          rFlagsReg cr) %{
10744   match(Set dst (XorL src1 m1));
10745   ins_cost(INSN_COST);
10746   format %{ "eon  $dst, $src1, zr" %}
10747 
10748   ins_encode %{
10749     __ eon(as_Register($dst$$reg),
10750               as_Register($src1$$reg),
10751               zr,
10752               Assembler::LSL, 0);
10753   %}
10754 
10755   ins_pipe(ialu_reg);
10756 %}
10757 
10758 // This pattern is automatically generated from aarch64_ad.m4
10759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10760 instruct regI_not_reg(iRegINoSp dst,
10761                          iRegIorL2I src1, immI_M1 m1,
10762                          rFlagsReg cr) %{
10763   match(Set dst (XorI src1 m1));
10764   ins_cost(INSN_COST);
10765   format %{ "eonw  $dst, $src1, zr" %}
10766 
10767   ins_encode %{
10768     __ eonw(as_Register($dst$$reg),
10769               as_Register($src1$$reg),
10770               zr,
10771               Assembler::LSL, 0);
10772   %}
10773 
10774   ins_pipe(ialu_reg);
10775 %}
10776 
10777 // This pattern is automatically generated from aarch64_ad.m4
10778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10779 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10780                               immI0 zero, iRegIorL2I src1, immI src2) %{
10781   match(Set dst (SubI zero (URShiftI src1 src2)));
10782 
10783   ins_cost(1.9 * INSN_COST);
10784   format %{ "negw  $dst, $src1, LSR $src2" %}
10785 
10786   ins_encode %{
10787     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10788             Assembler::LSR, $src2$$constant & 0x1f);
10789   %}
10790 
10791   ins_pipe(ialu_reg_shift);
10792 %}
10793 
10794 // This pattern is automatically generated from aarch64_ad.m4
10795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10796 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10797                               immI0 zero, iRegIorL2I src1, immI src2) %{
10798   match(Set dst (SubI zero (RShiftI src1 src2)));
10799 
10800   ins_cost(1.9 * INSN_COST);
10801   format %{ "negw  $dst, $src1, ASR $src2" %}
10802 
10803   ins_encode %{
10804     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10805             Assembler::ASR, $src2$$constant & 0x1f);
10806   %}
10807 
10808   ins_pipe(ialu_reg_shift);
10809 %}
10810 
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10814                               immI0 zero, iRegIorL2I src1, immI src2) %{
10815   match(Set dst (SubI zero (LShiftI src1 src2)));
10816 
10817   ins_cost(1.9 * INSN_COST);
10818   format %{ "negw  $dst, $src1, LSL $src2" %}
10819 
10820   ins_encode %{
10821     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10822             Assembler::LSL, $src2$$constant & 0x1f);
10823   %}
10824 
10825   ins_pipe(ialu_reg_shift);
10826 %}
10827 
10828 // This pattern is automatically generated from aarch64_ad.m4
10829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10830 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10831                               immL0 zero, iRegL src1, immI src2) %{
10832   match(Set dst (SubL zero (URShiftL src1 src2)));
10833 
10834   ins_cost(1.9 * INSN_COST);
10835   format %{ "neg  $dst, $src1, LSR $src2" %}
10836 
10837   ins_encode %{
10838     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10839             Assembler::LSR, $src2$$constant & 0x3f);
10840   %}
10841 
10842   ins_pipe(ialu_reg_shift);
10843 %}
10844 
10845 // This pattern is automatically generated from aarch64_ad.m4
10846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10847 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10848                               immL0 zero, iRegL src1, immI src2) %{
10849   match(Set dst (SubL zero (RShiftL src1 src2)));
10850 
10851   ins_cost(1.9 * INSN_COST);
10852   format %{ "neg  $dst, $src1, ASR $src2" %}
10853 
10854   ins_encode %{
10855     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10856             Assembler::ASR, $src2$$constant & 0x3f);
10857   %}
10858 
10859   ins_pipe(ialu_reg_shift);
10860 %}
10861 
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10865                               immL0 zero, iRegL src1, immI src2) %{
10866   match(Set dst (SubL zero (LShiftL src1 src2)));
10867 
10868   ins_cost(1.9 * INSN_COST);
10869   format %{ "neg  $dst, $src1, LSL $src2" %}
10870 
10871   ins_encode %{
10872     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10873             Assembler::LSL, $src2$$constant & 0x3f);
10874   %}
10875 
10876   ins_pipe(ialu_reg_shift);
10877 %}
10878 
10879 // This pattern is automatically generated from aarch64_ad.m4
10880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10881 instruct AndI_reg_not_reg(iRegINoSp dst,
10882                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10883   match(Set dst (AndI src1 (XorI src2 m1)));
10884   ins_cost(INSN_COST);
10885   format %{ "bicw  $dst, $src1, $src2" %}
10886 
10887   ins_encode %{
10888     __ bicw(as_Register($dst$$reg),
10889               as_Register($src1$$reg),
10890               as_Register($src2$$reg),
10891               Assembler::LSL, 0);
10892   %}
10893 
10894   ins_pipe(ialu_reg_reg);
10895 %}
10896 
10897 // This pattern is automatically generated from aarch64_ad.m4
10898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10899 instruct AndL_reg_not_reg(iRegLNoSp dst,
10900                          iRegL src1, iRegL src2, immL_M1 m1) %{
10901   match(Set dst (AndL src1 (XorL src2 m1)));
10902   ins_cost(INSN_COST);
10903   format %{ "bic  $dst, $src1, $src2" %}
10904 
10905   ins_encode %{
10906     __ bic(as_Register($dst$$reg),
10907               as_Register($src1$$reg),
10908               as_Register($src2$$reg),
10909               Assembler::LSL, 0);
10910   %}
10911 
10912   ins_pipe(ialu_reg_reg);
10913 %}
10914 
10915 // This pattern is automatically generated from aarch64_ad.m4
10916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10917 instruct OrI_reg_not_reg(iRegINoSp dst,
10918                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10919   match(Set dst (OrI src1 (XorI src2 m1)));
10920   ins_cost(INSN_COST);
10921   format %{ "ornw  $dst, $src1, $src2" %}
10922 
10923   ins_encode %{
10924     __ ornw(as_Register($dst$$reg),
10925               as_Register($src1$$reg),
10926               as_Register($src2$$reg),
10927               Assembler::LSL, 0);
10928   %}
10929 
10930   ins_pipe(ialu_reg_reg);
10931 %}
10932 
10933 // This pattern is automatically generated from aarch64_ad.m4
10934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10935 instruct OrL_reg_not_reg(iRegLNoSp dst,
10936                          iRegL src1, iRegL src2, immL_M1 m1) %{
10937   match(Set dst (OrL src1 (XorL src2 m1)));
10938   ins_cost(INSN_COST);
10939   format %{ "orn  $dst, $src1, $src2" %}
10940 
10941   ins_encode %{
10942     __ orn(as_Register($dst$$reg),
10943               as_Register($src1$$reg),
10944               as_Register($src2$$reg),
10945               Assembler::LSL, 0);
10946   %}
10947 
10948   ins_pipe(ialu_reg_reg);
10949 %}
10950 
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorI_reg_not_reg(iRegINoSp dst,
10954                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10955   match(Set dst (XorI m1 (XorI src2 src1)));
10956   ins_cost(INSN_COST);
10957   format %{ "eonw  $dst, $src1, $src2" %}
10958 
10959   ins_encode %{
10960     __ eonw(as_Register($dst$$reg),
10961               as_Register($src1$$reg),
10962               as_Register($src2$$reg),
10963               Assembler::LSL, 0);
10964   %}
10965 
10966   ins_pipe(ialu_reg_reg);
10967 %}
10968 
10969 // This pattern is automatically generated from aarch64_ad.m4
10970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10971 instruct XorL_reg_not_reg(iRegLNoSp dst,
10972                          iRegL src1, iRegL src2, immL_M1 m1) %{
10973   match(Set dst (XorL m1 (XorL src2 src1)));
10974   ins_cost(INSN_COST);
10975   format %{ "eon  $dst, $src1, $src2" %}
10976 
10977   ins_encode %{
10978     __ eon(as_Register($dst$$reg),
10979               as_Register($src1$$reg),
10980               as_Register($src2$$reg),
10981               Assembler::LSL, 0);
10982   %}
10983 
10984   ins_pipe(ialu_reg_reg);
10985 %}
10986 
10987 // This pattern is automatically generated from aarch64_ad.m4
10988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10989 // val & (-1 ^ (val >>> shift)) ==> bicw
10990 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10991                          iRegIorL2I src1, iRegIorL2I src2,
10992                          immI src3, immI_M1 src4) %{
10993   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10994   ins_cost(1.9 * INSN_COST);
10995   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10996 
10997   ins_encode %{
10998     __ bicw(as_Register($dst$$reg),
10999               as_Register($src1$$reg),
11000               as_Register($src2$$reg),
11001               Assembler::LSR,
11002               $src3$$constant & 0x1f);
11003   %}
11004 
11005   ins_pipe(ialu_reg_reg_shift);
11006 %}
11007 
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 // val & (-1 ^ (val >>> shift)) ==> bic
11011 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11012                          iRegL src1, iRegL src2,
11013                          immI src3, immL_M1 src4) %{
11014   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11015   ins_cost(1.9 * INSN_COST);
11016   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11017 
11018   ins_encode %{
11019     __ bic(as_Register($dst$$reg),
11020               as_Register($src1$$reg),
11021               as_Register($src2$$reg),
11022               Assembler::LSR,
11023               $src3$$constant & 0x3f);
11024   %}
11025 
11026   ins_pipe(ialu_reg_reg_shift);
11027 %}
11028 
11029 // This pattern is automatically generated from aarch64_ad.m4
11030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11031 // val & (-1 ^ (val >> shift)) ==> bicw
11032 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11033                          iRegIorL2I src1, iRegIorL2I src2,
11034                          immI src3, immI_M1 src4) %{
11035   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11036   ins_cost(1.9 * INSN_COST);
11037   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11038 
11039   ins_encode %{
11040     __ bicw(as_Register($dst$$reg),
11041               as_Register($src1$$reg),
11042               as_Register($src2$$reg),
11043               Assembler::ASR,
11044               $src3$$constant & 0x1f);
11045   %}
11046 
11047   ins_pipe(ialu_reg_reg_shift);
11048 %}
11049 
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 // val & (-1 ^ (val >> shift)) ==> bic
11053 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11054                          iRegL src1, iRegL src2,
11055                          immI src3, immL_M1 src4) %{
11056   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11057   ins_cost(1.9 * INSN_COST);
11058   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11059 
11060   ins_encode %{
11061     __ bic(as_Register($dst$$reg),
11062               as_Register($src1$$reg),
11063               as_Register($src2$$reg),
11064               Assembler::ASR,
11065               $src3$$constant & 0x3f);
11066   %}
11067 
11068   ins_pipe(ialu_reg_reg_shift);
11069 %}
11070 
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 // val & (-1 ^ (val ror shift)) ==> bicw
11074 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11075                          iRegIorL2I src1, iRegIorL2I src2,
11076                          immI src3, immI_M1 src4) %{
11077   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11078   ins_cost(1.9 * INSN_COST);
11079   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11080 
11081   ins_encode %{
11082     __ bicw(as_Register($dst$$reg),
11083               as_Register($src1$$reg),
11084               as_Register($src2$$reg),
11085               Assembler::ROR,
11086               $src3$$constant & 0x1f);
11087   %}
11088 
11089   ins_pipe(ialu_reg_reg_shift);
11090 %}
11091 
11092 // This pattern is automatically generated from aarch64_ad.m4
11093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11094 // val & (-1 ^ (val ror shift)) ==> bic
11095 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11096                          iRegL src1, iRegL src2,
11097                          immI src3, immL_M1 src4) %{
11098   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11099   ins_cost(1.9 * INSN_COST);
11100   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11101 
11102   ins_encode %{
11103     __ bic(as_Register($dst$$reg),
11104               as_Register($src1$$reg),
11105               as_Register($src2$$reg),
11106               Assembler::ROR,
11107               $src3$$constant & 0x3f);
11108   %}
11109 
11110   ins_pipe(ialu_reg_reg_shift);
11111 %}
11112 
11113 // This pattern is automatically generated from aarch64_ad.m4
11114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11115 // val & (-1 ^ (val << shift)) ==> bicw
11116 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11117                          iRegIorL2I src1, iRegIorL2I src2,
11118                          immI src3, immI_M1 src4) %{
11119   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11120   ins_cost(1.9 * INSN_COST);
11121   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11122 
11123   ins_encode %{
11124     __ bicw(as_Register($dst$$reg),
11125               as_Register($src1$$reg),
11126               as_Register($src2$$reg),
11127               Assembler::LSL,
11128               $src3$$constant & 0x1f);
11129   %}
11130 
11131   ins_pipe(ialu_reg_reg_shift);
11132 %}
11133 
11134 // This pattern is automatically generated from aarch64_ad.m4
11135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11136 // val & (-1 ^ (val << shift)) ==> bic
11137 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11138                          iRegL src1, iRegL src2,
11139                          immI src3, immL_M1 src4) %{
11140   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11141   ins_cost(1.9 * INSN_COST);
11142   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11143 
11144   ins_encode %{
11145     __ bic(as_Register($dst$$reg),
11146               as_Register($src1$$reg),
11147               as_Register($src2$$reg),
11148               Assembler::LSL,
11149               $src3$$constant & 0x3f);
11150   %}
11151 
11152   ins_pipe(ialu_reg_reg_shift);
11153 %}
11154 
11155 // This pattern is automatically generated from aarch64_ad.m4
11156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11157 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11158 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11159                          iRegIorL2I src1, iRegIorL2I src2,
11160                          immI src3, immI_M1 src4) %{
11161   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11162   ins_cost(1.9 * INSN_COST);
11163   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11164 
11165   ins_encode %{
11166     __ eonw(as_Register($dst$$reg),
11167               as_Register($src1$$reg),
11168               as_Register($src2$$reg),
11169               Assembler::LSR,
11170               $src3$$constant & 0x1f);
11171   %}
11172 
11173   ins_pipe(ialu_reg_reg_shift);
11174 %}
11175 
11176 // This pattern is automatically generated from aarch64_ad.m4
11177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11178 // val ^ (-1 ^ (val >>> shift)) ==> eon
11179 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11180                          iRegL src1, iRegL src2,
11181                          immI src3, immL_M1 src4) %{
11182   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11183   ins_cost(1.9 * INSN_COST);
11184   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11185 
11186   ins_encode %{
11187     __ eon(as_Register($dst$$reg),
11188               as_Register($src1$$reg),
11189               as_Register($src2$$reg),
11190               Assembler::LSR,
11191               $src3$$constant & 0x3f);
11192   %}
11193 
11194   ins_pipe(ialu_reg_reg_shift);
11195 %}
11196 
11197 // This pattern is automatically generated from aarch64_ad.m4
11198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11199 // val ^ (-1 ^ (val >> shift)) ==> eonw
11200 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11201                          iRegIorL2I src1, iRegIorL2I src2,
11202                          immI src3, immI_M1 src4) %{
11203   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11204   ins_cost(1.9 * INSN_COST);
11205   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11206 
11207   ins_encode %{
11208     __ eonw(as_Register($dst$$reg),
11209               as_Register($src1$$reg),
11210               as_Register($src2$$reg),
11211               Assembler::ASR,
11212               $src3$$constant & 0x1f);
11213   %}
11214 
11215   ins_pipe(ialu_reg_reg_shift);
11216 %}
11217 
11218 // This pattern is automatically generated from aarch64_ad.m4
11219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11220 // val ^ (-1 ^ (val >> shift)) ==> eon
11221 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11222                          iRegL src1, iRegL src2,
11223                          immI src3, immL_M1 src4) %{
11224   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11225   ins_cost(1.9 * INSN_COST);
11226   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11227 
11228   ins_encode %{
11229     __ eon(as_Register($dst$$reg),
11230               as_Register($src1$$reg),
11231               as_Register($src2$$reg),
11232               Assembler::ASR,
11233               $src3$$constant & 0x3f);
11234   %}
11235 
11236   ins_pipe(ialu_reg_reg_shift);
11237 %}
11238 
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 // val ^ (-1 ^ (val ror shift)) ==> eonw
11242 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11243                          iRegIorL2I src1, iRegIorL2I src2,
11244                          immI src3, immI_M1 src4) %{
11245   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11246   ins_cost(1.9 * INSN_COST);
11247   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11248 
11249   ins_encode %{
11250     __ eonw(as_Register($dst$$reg),
11251               as_Register($src1$$reg),
11252               as_Register($src2$$reg),
11253               Assembler::ROR,
11254               $src3$$constant & 0x1f);
11255   %}
11256 
11257   ins_pipe(ialu_reg_reg_shift);
11258 %}
11259 
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 // val ^ (-1 ^ (val ror shift)) ==> eon
11263 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11264                          iRegL src1, iRegL src2,
11265                          immI src3, immL_M1 src4) %{
11266   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11267   ins_cost(1.9 * INSN_COST);
11268   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11269 
11270   ins_encode %{
11271     __ eon(as_Register($dst$$reg),
11272               as_Register($src1$$reg),
11273               as_Register($src2$$reg),
11274               Assembler::ROR,
11275               $src3$$constant & 0x3f);
11276   %}
11277 
11278   ins_pipe(ialu_reg_reg_shift);
11279 %}
11280 
11281 // This pattern is automatically generated from aarch64_ad.m4
11282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11283 // val ^ (-1 ^ (val << shift)) ==> eonw
11284 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11285                          iRegIorL2I src1, iRegIorL2I src2,
11286                          immI src3, immI_M1 src4) %{
11287   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11288   ins_cost(1.9 * INSN_COST);
11289   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11290 
11291   ins_encode %{
11292     __ eonw(as_Register($dst$$reg),
11293               as_Register($src1$$reg),
11294               as_Register($src2$$reg),
11295               Assembler::LSL,
11296               $src3$$constant & 0x1f);
11297   %}
11298 
11299   ins_pipe(ialu_reg_reg_shift);
11300 %}
11301 
11302 // This pattern is automatically generated from aarch64_ad.m4
11303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11304 // val ^ (-1 ^ (val << shift)) ==> eon
11305 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11306                          iRegL src1, iRegL src2,
11307                          immI src3, immL_M1 src4) %{
11308   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11309   ins_cost(1.9 * INSN_COST);
11310   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11311 
11312   ins_encode %{
11313     __ eon(as_Register($dst$$reg),
11314               as_Register($src1$$reg),
11315               as_Register($src2$$reg),
11316               Assembler::LSL,
11317               $src3$$constant & 0x3f);
11318   %}
11319 
11320   ins_pipe(ialu_reg_reg_shift);
11321 %}
11322 
11323 // This pattern is automatically generated from aarch64_ad.m4
11324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11325 // val | (-1 ^ (val >>> shift)) ==> ornw
11326 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11327                          iRegIorL2I src1, iRegIorL2I src2,
11328                          immI src3, immI_M1 src4) %{
11329   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11330   ins_cost(1.9 * INSN_COST);
11331   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11332 
11333   ins_encode %{
11334     __ ornw(as_Register($dst$$reg),
11335               as_Register($src1$$reg),
11336               as_Register($src2$$reg),
11337               Assembler::LSR,
11338               $src3$$constant & 0x1f);
11339   %}
11340 
11341   ins_pipe(ialu_reg_reg_shift);
11342 %}
11343 
11344 // This pattern is automatically generated from aarch64_ad.m4
11345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11346 // val | (-1 ^ (val >>> shift)) ==> orn
11347 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11348                          iRegL src1, iRegL src2,
11349                          immI src3, immL_M1 src4) %{
11350   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11351   ins_cost(1.9 * INSN_COST);
11352   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11353 
11354   ins_encode %{
11355     __ orn(as_Register($dst$$reg),
11356               as_Register($src1$$reg),
11357               as_Register($src2$$reg),
11358               Assembler::LSR,
11359               $src3$$constant & 0x3f);
11360   %}
11361 
11362   ins_pipe(ialu_reg_reg_shift);
11363 %}
11364 
11365 // This pattern is automatically generated from aarch64_ad.m4
11366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11367 // val | (-1 ^ (val >> shift)) ==> ornw
11368 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11369                          iRegIorL2I src1, iRegIorL2I src2,
11370                          immI src3, immI_M1 src4) %{
11371   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11372   ins_cost(1.9 * INSN_COST);
11373   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11374 
11375   ins_encode %{
11376     __ ornw(as_Register($dst$$reg),
11377               as_Register($src1$$reg),
11378               as_Register($src2$$reg),
11379               Assembler::ASR,
11380               $src3$$constant & 0x1f);
11381   %}
11382 
11383   ins_pipe(ialu_reg_reg_shift);
11384 %}
11385 
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388 // val | (-1 ^ (val >> shift)) ==> orn
11389 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11390                          iRegL src1, iRegL src2,
11391                          immI src3, immL_M1 src4) %{
11392   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11393   ins_cost(1.9 * INSN_COST);
11394   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11395 
11396   ins_encode %{
11397     __ orn(as_Register($dst$$reg),
11398               as_Register($src1$$reg),
11399               as_Register($src2$$reg),
11400               Assembler::ASR,
11401               $src3$$constant & 0x3f);
11402   %}
11403 
11404   ins_pipe(ialu_reg_reg_shift);
11405 %}
11406 
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409 // val | (-1 ^ (val ror shift)) ==> ornw
11410 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11411                          iRegIorL2I src1, iRegIorL2I src2,
11412                          immI src3, immI_M1 src4) %{
11413   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11414   ins_cost(1.9 * INSN_COST);
11415   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11416 
11417   ins_encode %{
11418     __ ornw(as_Register($dst$$reg),
11419               as_Register($src1$$reg),
11420               as_Register($src2$$reg),
11421               Assembler::ROR,
11422               $src3$$constant & 0x1f);
11423   %}
11424 
11425   ins_pipe(ialu_reg_reg_shift);
11426 %}
11427 
11428 // This pattern is automatically generated from aarch64_ad.m4
11429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11430 // val | (-1 ^ (val ror shift)) ==> orn
11431 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11432                          iRegL src1, iRegL src2,
11433                          immI src3, immL_M1 src4) %{
11434   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11435   ins_cost(1.9 * INSN_COST);
11436   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11437 
11438   ins_encode %{
11439     __ orn(as_Register($dst$$reg),
11440               as_Register($src1$$reg),
11441               as_Register($src2$$reg),
11442               Assembler::ROR,
11443               $src3$$constant & 0x3f);
11444   %}
11445 
11446   ins_pipe(ialu_reg_reg_shift);
11447 %}
11448 
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 // val | (-1 ^ (val << shift)) ==> ornw
11452 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11453                          iRegIorL2I src1, iRegIorL2I src2,
11454                          immI src3, immI_M1 src4) %{
11455   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11456   ins_cost(1.9 * INSN_COST);
11457   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11458 
11459   ins_encode %{
11460     __ ornw(as_Register($dst$$reg),
11461               as_Register($src1$$reg),
11462               as_Register($src2$$reg),
11463               Assembler::LSL,
11464               $src3$$constant & 0x1f);
11465   %}
11466 
11467   ins_pipe(ialu_reg_reg_shift);
11468 %}
11469 
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 // val | (-1 ^ (val << shift)) ==> orn
11473 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11474                          iRegL src1, iRegL src2,
11475                          immI src3, immL_M1 src4) %{
11476   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11477   ins_cost(1.9 * INSN_COST);
11478   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11479 
11480   ins_encode %{
11481     __ orn(as_Register($dst$$reg),
11482               as_Register($src1$$reg),
11483               as_Register($src2$$reg),
11484               Assembler::LSL,
11485               $src3$$constant & 0x3f);
11486   %}
11487 
11488   ins_pipe(ialu_reg_reg_shift);
11489 %}
11490 
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11494                          iRegIorL2I src1, iRegIorL2I src2,
11495                          immI src3) %{
11496   match(Set dst (AndI src1 (URShiftI src2 src3)));
11497 
11498   ins_cost(1.9 * INSN_COST);
11499   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11500 
11501   ins_encode %{
11502     __ andw(as_Register($dst$$reg),
11503               as_Register($src1$$reg),
11504               as_Register($src2$$reg),
11505               Assembler::LSR,
11506               $src3$$constant & 0x1f);
11507   %}
11508 
11509   ins_pipe(ialu_reg_reg_shift);
11510 %}
11511 
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11515                          iRegL src1, iRegL src2,
11516                          immI src3) %{
11517   match(Set dst (AndL src1 (URShiftL src2 src3)));
11518 
11519   ins_cost(1.9 * INSN_COST);
11520   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11521 
11522   ins_encode %{
11523     __ andr(as_Register($dst$$reg),
11524               as_Register($src1$$reg),
11525               as_Register($src2$$reg),
11526               Assembler::LSR,
11527               $src3$$constant & 0x3f);
11528   %}
11529 
11530   ins_pipe(ialu_reg_reg_shift);
11531 %}
11532 
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11536                          iRegIorL2I src1, iRegIorL2I src2,
11537                          immI src3) %{
11538   match(Set dst (AndI src1 (RShiftI src2 src3)));
11539 
11540   ins_cost(1.9 * INSN_COST);
11541   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11542 
11543   ins_encode %{
11544     __ andw(as_Register($dst$$reg),
11545               as_Register($src1$$reg),
11546               as_Register($src2$$reg),
11547               Assembler::ASR,
11548               $src3$$constant & 0x1f);
11549   %}
11550 
11551   ins_pipe(ialu_reg_reg_shift);
11552 %}
11553 
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11557                          iRegL src1, iRegL src2,
11558                          immI src3) %{
11559   match(Set dst (AndL src1 (RShiftL src2 src3)));
11560 
11561   ins_cost(1.9 * INSN_COST);
11562   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11563 
11564   ins_encode %{
11565     __ andr(as_Register($dst$$reg),
11566               as_Register($src1$$reg),
11567               as_Register($src2$$reg),
11568               Assembler::ASR,
11569               $src3$$constant & 0x3f);
11570   %}
11571 
11572   ins_pipe(ialu_reg_reg_shift);
11573 %}
11574 
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11578                          iRegIorL2I src1, iRegIorL2I src2,
11579                          immI src3) %{
11580   match(Set dst (AndI src1 (LShiftI src2 src3)));
11581 
11582   ins_cost(1.9 * INSN_COST);
11583   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11584 
11585   ins_encode %{
11586     __ andw(as_Register($dst$$reg),
11587               as_Register($src1$$reg),
11588               as_Register($src2$$reg),
11589               Assembler::LSL,
11590               $src3$$constant & 0x1f);
11591   %}
11592 
11593   ins_pipe(ialu_reg_reg_shift);
11594 %}
11595 
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11599                          iRegL src1, iRegL src2,
11600                          immI src3) %{
11601   match(Set dst (AndL src1 (LShiftL src2 src3)));
11602 
11603   ins_cost(1.9 * INSN_COST);
11604   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11605 
11606   ins_encode %{
11607     __ andr(as_Register($dst$$reg),
11608               as_Register($src1$$reg),
11609               as_Register($src2$$reg),
11610               Assembler::LSL,
11611               $src3$$constant & 0x3f);
11612   %}
11613 
11614   ins_pipe(ialu_reg_reg_shift);
11615 %}
11616 
11617 // This pattern is automatically generated from aarch64_ad.m4
11618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11619 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11620                          iRegIorL2I src1, iRegIorL2I src2,
11621                          immI src3) %{
11622   match(Set dst (AndI src1 (RotateRight src2 src3)));
11623 
11624   ins_cost(1.9 * INSN_COST);
11625   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11626 
11627   ins_encode %{
11628     __ andw(as_Register($dst$$reg),
11629               as_Register($src1$$reg),
11630               as_Register($src2$$reg),
11631               Assembler::ROR,
11632               $src3$$constant & 0x1f);
11633   %}
11634 
11635   ins_pipe(ialu_reg_reg_shift);
11636 %}
11637 
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11641                          iRegL src1, iRegL src2,
11642                          immI src3) %{
11643   match(Set dst (AndL src1 (RotateRight src2 src3)));
11644 
11645   ins_cost(1.9 * INSN_COST);
11646   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11647 
11648   ins_encode %{
11649     __ andr(as_Register($dst$$reg),
11650               as_Register($src1$$reg),
11651               as_Register($src2$$reg),
11652               Assembler::ROR,
11653               $src3$$constant & 0x3f);
11654   %}
11655 
11656   ins_pipe(ialu_reg_reg_shift);
11657 %}
11658 
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11662                          iRegIorL2I src1, iRegIorL2I src2,
11663                          immI src3) %{
11664   match(Set dst (XorI src1 (URShiftI src2 src3)));
11665 
11666   ins_cost(1.9 * INSN_COST);
11667   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11668 
11669   ins_encode %{
11670     __ eorw(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::LSR,
11674               $src3$$constant & 0x1f);
11675   %}
11676 
11677   ins_pipe(ialu_reg_reg_shift);
11678 %}
11679 
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11683                          iRegL src1, iRegL src2,
11684                          immI src3) %{
11685   match(Set dst (XorL src1 (URShiftL src2 src3)));
11686 
11687   ins_cost(1.9 * INSN_COST);
11688   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11689 
11690   ins_encode %{
11691     __ eor(as_Register($dst$$reg),
11692               as_Register($src1$$reg),
11693               as_Register($src2$$reg),
11694               Assembler::LSR,
11695               $src3$$constant & 0x3f);
11696   %}
11697 
11698   ins_pipe(ialu_reg_reg_shift);
11699 %}
11700 
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11704                          iRegIorL2I src1, iRegIorL2I src2,
11705                          immI src3) %{
11706   match(Set dst (XorI src1 (RShiftI src2 src3)));
11707 
11708   ins_cost(1.9 * INSN_COST);
11709   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11710 
11711   ins_encode %{
11712     __ eorw(as_Register($dst$$reg),
11713               as_Register($src1$$reg),
11714               as_Register($src2$$reg),
11715               Assembler::ASR,
11716               $src3$$constant & 0x1f);
11717   %}
11718 
11719   ins_pipe(ialu_reg_reg_shift);
11720 %}
11721 
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11725                          iRegL src1, iRegL src2,
11726                          immI src3) %{
11727   match(Set dst (XorL src1 (RShiftL src2 src3)));
11728 
11729   ins_cost(1.9 * INSN_COST);
11730   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11731 
11732   ins_encode %{
11733     __ eor(as_Register($dst$$reg),
11734               as_Register($src1$$reg),
11735               as_Register($src2$$reg),
11736               Assembler::ASR,
11737               $src3$$constant & 0x3f);
11738   %}
11739 
11740   ins_pipe(ialu_reg_reg_shift);
11741 %}
11742 
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11746                          iRegIorL2I src1, iRegIorL2I src2,
11747                          immI src3) %{
11748   match(Set dst (XorI src1 (LShiftI src2 src3)));
11749 
11750   ins_cost(1.9 * INSN_COST);
11751   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11752 
11753   ins_encode %{
11754     __ eorw(as_Register($dst$$reg),
11755               as_Register($src1$$reg),
11756               as_Register($src2$$reg),
11757               Assembler::LSL,
11758               $src3$$constant & 0x1f);
11759   %}
11760 
11761   ins_pipe(ialu_reg_reg_shift);
11762 %}
11763 
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11767                          iRegL src1, iRegL src2,
11768                          immI src3) %{
11769   match(Set dst (XorL src1 (LShiftL src2 src3)));
11770 
11771   ins_cost(1.9 * INSN_COST);
11772   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11773 
11774   ins_encode %{
11775     __ eor(as_Register($dst$$reg),
11776               as_Register($src1$$reg),
11777               as_Register($src2$$reg),
11778               Assembler::LSL,
11779               $src3$$constant & 0x3f);
11780   %}
11781 
11782   ins_pipe(ialu_reg_reg_shift);
11783 %}
11784 
11785 // This pattern is automatically generated from aarch64_ad.m4
11786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11787 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11788                          iRegIorL2I src1, iRegIorL2I src2,
11789                          immI src3) %{
11790   match(Set dst (XorI src1 (RotateRight src2 src3)));
11791 
11792   ins_cost(1.9 * INSN_COST);
11793   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11794 
11795   ins_encode %{
11796     __ eorw(as_Register($dst$$reg),
11797               as_Register($src1$$reg),
11798               as_Register($src2$$reg),
11799               Assembler::ROR,
11800               $src3$$constant & 0x1f);
11801   %}
11802 
11803   ins_pipe(ialu_reg_reg_shift);
11804 %}
11805 
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11809                          iRegL src1, iRegL src2,
11810                          immI src3) %{
11811   match(Set dst (XorL src1 (RotateRight src2 src3)));
11812 
11813   ins_cost(1.9 * INSN_COST);
11814   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11815 
11816   ins_encode %{
11817     __ eor(as_Register($dst$$reg),
11818               as_Register($src1$$reg),
11819               as_Register($src2$$reg),
11820               Assembler::ROR,
11821               $src3$$constant & 0x3f);
11822   %}
11823 
11824   ins_pipe(ialu_reg_reg_shift);
11825 %}
11826 
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11830                          iRegIorL2I src1, iRegIorL2I src2,
11831                          immI src3) %{
11832   match(Set dst (OrI src1 (URShiftI src2 src3)));
11833 
11834   ins_cost(1.9 * INSN_COST);
11835   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11836 
11837   ins_encode %{
11838     __ orrw(as_Register($dst$$reg),
11839               as_Register($src1$$reg),
11840               as_Register($src2$$reg),
11841               Assembler::LSR,
11842               $src3$$constant & 0x1f);
11843   %}
11844 
11845   ins_pipe(ialu_reg_reg_shift);
11846 %}
11847 
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11851                          iRegL src1, iRegL src2,
11852                          immI src3) %{
11853   match(Set dst (OrL src1 (URShiftL src2 src3)));
11854 
11855   ins_cost(1.9 * INSN_COST);
11856   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11857 
11858   ins_encode %{
11859     __ orr(as_Register($dst$$reg),
11860               as_Register($src1$$reg),
11861               as_Register($src2$$reg),
11862               Assembler::LSR,
11863               $src3$$constant & 0x3f);
11864   %}
11865 
11866   ins_pipe(ialu_reg_reg_shift);
11867 %}
11868 
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11872                          iRegIorL2I src1, iRegIorL2I src2,
11873                          immI src3) %{
11874   match(Set dst (OrI src1 (RShiftI src2 src3)));
11875 
11876   ins_cost(1.9 * INSN_COST);
11877   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11878 
11879   ins_encode %{
11880     __ orrw(as_Register($dst$$reg),
11881               as_Register($src1$$reg),
11882               as_Register($src2$$reg),
11883               Assembler::ASR,
11884               $src3$$constant & 0x1f);
11885   %}
11886 
11887   ins_pipe(ialu_reg_reg_shift);
11888 %}
11889 
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11893                          iRegL src1, iRegL src2,
11894                          immI src3) %{
11895   match(Set dst (OrL src1 (RShiftL src2 src3)));
11896 
11897   ins_cost(1.9 * INSN_COST);
11898   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11899 
11900   ins_encode %{
11901     __ orr(as_Register($dst$$reg),
11902               as_Register($src1$$reg),
11903               as_Register($src2$$reg),
11904               Assembler::ASR,
11905               $src3$$constant & 0x3f);
11906   %}
11907 
11908   ins_pipe(ialu_reg_reg_shift);
11909 %}
11910 
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11914                          iRegIorL2I src1, iRegIorL2I src2,
11915                          immI src3) %{
11916   match(Set dst (OrI src1 (LShiftI src2 src3)));
11917 
11918   ins_cost(1.9 * INSN_COST);
11919   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11920 
11921   ins_encode %{
11922     __ orrw(as_Register($dst$$reg),
11923               as_Register($src1$$reg),
11924               as_Register($src2$$reg),
11925               Assembler::LSL,
11926               $src3$$constant & 0x1f);
11927   %}
11928 
11929   ins_pipe(ialu_reg_reg_shift);
11930 %}
11931 
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11935                          iRegL src1, iRegL src2,
11936                          immI src3) %{
11937   match(Set dst (OrL src1 (LShiftL src2 src3)));
11938 
11939   ins_cost(1.9 * INSN_COST);
11940   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11941 
11942   ins_encode %{
11943     __ orr(as_Register($dst$$reg),
11944               as_Register($src1$$reg),
11945               as_Register($src2$$reg),
11946               Assembler::LSL,
11947               $src3$$constant & 0x3f);
11948   %}
11949 
11950   ins_pipe(ialu_reg_reg_shift);
11951 %}
11952 
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11956                          iRegIorL2I src1, iRegIorL2I src2,
11957                          immI src3) %{
11958   match(Set dst (OrI src1 (RotateRight src2 src3)));
11959 
11960   ins_cost(1.9 * INSN_COST);
11961   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
11962 
11963   ins_encode %{
11964     __ orrw(as_Register($dst$$reg),
11965               as_Register($src1$$reg),
11966               as_Register($src2$$reg),
11967               Assembler::ROR,
11968               $src3$$constant & 0x1f);
11969   %}
11970 
11971   ins_pipe(ialu_reg_reg_shift);
11972 %}
11973 
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11977                          iRegL src1, iRegL src2,
11978                          immI src3) %{
11979   match(Set dst (OrL src1 (RotateRight src2 src3)));
11980 
11981   ins_cost(1.9 * INSN_COST);
11982   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
11983 
11984   ins_encode %{
11985     __ orr(as_Register($dst$$reg),
11986               as_Register($src1$$reg),
11987               as_Register($src2$$reg),
11988               Assembler::ROR,
11989               $src3$$constant & 0x3f);
11990   %}
11991 
11992   ins_pipe(ialu_reg_reg_shift);
11993 %}
11994 
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11998                          iRegIorL2I src1, iRegIorL2I src2,
11999                          immI src3) %{
12000   match(Set dst (AddI src1 (URShiftI src2 src3)));
12001 
12002   ins_cost(1.9 * INSN_COST);
12003   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12004 
12005   ins_encode %{
12006     __ addw(as_Register($dst$$reg),
12007               as_Register($src1$$reg),
12008               as_Register($src2$$reg),
12009               Assembler::LSR,
12010               $src3$$constant & 0x1f);
12011   %}
12012 
12013   ins_pipe(ialu_reg_reg_shift);
12014 %}
12015 
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12019                          iRegL src1, iRegL src2,
12020                          immI src3) %{
12021   match(Set dst (AddL src1 (URShiftL src2 src3)));
12022 
12023   ins_cost(1.9 * INSN_COST);
12024   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12025 
12026   ins_encode %{
12027     __ add(as_Register($dst$$reg),
12028               as_Register($src1$$reg),
12029               as_Register($src2$$reg),
12030               Assembler::LSR,
12031               $src3$$constant & 0x3f);
12032   %}
12033 
12034   ins_pipe(ialu_reg_reg_shift);
12035 %}
12036 
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12040                          iRegIorL2I src1, iRegIorL2I src2,
12041                          immI src3) %{
12042   match(Set dst (AddI src1 (RShiftI src2 src3)));
12043 
12044   ins_cost(1.9 * INSN_COST);
12045   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12046 
12047   ins_encode %{
12048     __ addw(as_Register($dst$$reg),
12049               as_Register($src1$$reg),
12050               as_Register($src2$$reg),
12051               Assembler::ASR,
12052               $src3$$constant & 0x1f);
12053   %}
12054 
12055   ins_pipe(ialu_reg_reg_shift);
12056 %}
12057 
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12061                          iRegL src1, iRegL src2,
12062                          immI src3) %{
12063   match(Set dst (AddL src1 (RShiftL src2 src3)));
12064 
12065   ins_cost(1.9 * INSN_COST);
12066   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12067 
12068   ins_encode %{
12069     __ add(as_Register($dst$$reg),
12070               as_Register($src1$$reg),
12071               as_Register($src2$$reg),
12072               Assembler::ASR,
12073               $src3$$constant & 0x3f);
12074   %}
12075 
12076   ins_pipe(ialu_reg_reg_shift);
12077 %}
12078 
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12082                          iRegIorL2I src1, iRegIorL2I src2,
12083                          immI src3) %{
12084   match(Set dst (AddI src1 (LShiftI src2 src3)));
12085 
12086   ins_cost(1.9 * INSN_COST);
12087   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12088 
12089   ins_encode %{
12090     __ addw(as_Register($dst$$reg),
12091               as_Register($src1$$reg),
12092               as_Register($src2$$reg),
12093               Assembler::LSL,
12094               $src3$$constant & 0x1f);
12095   %}
12096 
12097   ins_pipe(ialu_reg_reg_shift);
12098 %}
12099 
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12103                          iRegL src1, iRegL src2,
12104                          immI src3) %{
12105   match(Set dst (AddL src1 (LShiftL src2 src3)));
12106 
12107   ins_cost(1.9 * INSN_COST);
12108   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12109 
12110   ins_encode %{
12111     __ add(as_Register($dst$$reg),
12112               as_Register($src1$$reg),
12113               as_Register($src2$$reg),
12114               Assembler::LSL,
12115               $src3$$constant & 0x3f);
12116   %}
12117 
12118   ins_pipe(ialu_reg_reg_shift);
12119 %}
12120 
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12124                          iRegIorL2I src1, iRegIorL2I src2,
12125                          immI src3) %{
12126   match(Set dst (SubI src1 (URShiftI src2 src3)));
12127 
12128   ins_cost(1.9 * INSN_COST);
12129   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12130 
12131   ins_encode %{
12132     __ subw(as_Register($dst$$reg),
12133               as_Register($src1$$reg),
12134               as_Register($src2$$reg),
12135               Assembler::LSR,
12136               $src3$$constant & 0x1f);
12137   %}
12138 
12139   ins_pipe(ialu_reg_reg_shift);
12140 %}
12141 
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12145                          iRegL src1, iRegL src2,
12146                          immI src3) %{
12147   match(Set dst (SubL src1 (URShiftL src2 src3)));
12148 
12149   ins_cost(1.9 * INSN_COST);
12150   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12151 
12152   ins_encode %{
12153     __ sub(as_Register($dst$$reg),
12154               as_Register($src1$$reg),
12155               as_Register($src2$$reg),
12156               Assembler::LSR,
12157               $src3$$constant & 0x3f);
12158   %}
12159 
12160   ins_pipe(ialu_reg_reg_shift);
12161 %}
12162 
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12166                          iRegIorL2I src1, iRegIorL2I src2,
12167                          immI src3) %{
12168   match(Set dst (SubI src1 (RShiftI src2 src3)));
12169 
12170   ins_cost(1.9 * INSN_COST);
12171   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12172 
12173   ins_encode %{
12174     __ subw(as_Register($dst$$reg),
12175               as_Register($src1$$reg),
12176               as_Register($src2$$reg),
12177               Assembler::ASR,
12178               $src3$$constant & 0x1f);
12179   %}
12180 
12181   ins_pipe(ialu_reg_reg_shift);
12182 %}
12183 
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12187                          iRegL src1, iRegL src2,
12188                          immI src3) %{
12189   match(Set dst (SubL src1 (RShiftL src2 src3)));
12190 
12191   ins_cost(1.9 * INSN_COST);
12192   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12193 
12194   ins_encode %{
12195     __ sub(as_Register($dst$$reg),
12196               as_Register($src1$$reg),
12197               as_Register($src2$$reg),
12198               Assembler::ASR,
12199               $src3$$constant & 0x3f);
12200   %}
12201 
12202   ins_pipe(ialu_reg_reg_shift);
12203 %}
12204 
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12208                          iRegIorL2I src1, iRegIorL2I src2,
12209                          immI src3) %{
12210   match(Set dst (SubI src1 (LShiftI src2 src3)));
12211 
12212   ins_cost(1.9 * INSN_COST);
12213   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12214 
12215   ins_encode %{
12216     __ subw(as_Register($dst$$reg),
12217               as_Register($src1$$reg),
12218               as_Register($src2$$reg),
12219               Assembler::LSL,
12220               $src3$$constant & 0x1f);
12221   %}
12222 
12223   ins_pipe(ialu_reg_reg_shift);
12224 %}
12225 
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12229                          iRegL src1, iRegL src2,
12230                          immI src3) %{
12231   match(Set dst (SubL src1 (LShiftL src2 src3)));
12232 
12233   ins_cost(1.9 * INSN_COST);
12234   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12235 
12236   ins_encode %{
12237     __ sub(as_Register($dst$$reg),
12238               as_Register($src1$$reg),
12239               as_Register($src2$$reg),
12240               Assembler::LSL,
12241               $src3$$constant & 0x3f);
12242   %}
12243 
12244   ins_pipe(ialu_reg_reg_shift);
12245 %}
12246 
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 
12250 // Shift Left followed by Shift Right.
12251 // This idiom is used by the compiler for the i2b bytecode etc.
12252 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12253 %{
12254   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12255   ins_cost(INSN_COST * 2);
12256   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12257   ins_encode %{
12258     int lshift = $lshift_count$$constant & 63;
12259     int rshift = $rshift_count$$constant & 63;
12260     int s = 63 - lshift;
12261     int r = (rshift - lshift) & 63;
12262     __ sbfm(as_Register($dst$$reg),
12263             as_Register($src$$reg),
12264             r, s);
12265   %}
12266 
12267   ins_pipe(ialu_reg_shift);
12268 %}
12269 
12270 // This pattern is automatically generated from aarch64_ad.m4
12271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12272 
12273 // Shift Left followed by Shift Right.
12274 // This idiom is used by the compiler for the i2b bytecode etc.
12275 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12276 %{
12277   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12278   ins_cost(INSN_COST * 2);
12279   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12280   ins_encode %{
12281     int lshift = $lshift_count$$constant & 31;
12282     int rshift = $rshift_count$$constant & 31;
12283     int s = 31 - lshift;
12284     int r = (rshift - lshift) & 31;
12285     __ sbfmw(as_Register($dst$$reg),
12286             as_Register($src$$reg),
12287             r, s);
12288   %}
12289 
12290   ins_pipe(ialu_reg_shift);
12291 %}
12292 
12293 // This pattern is automatically generated from aarch64_ad.m4
12294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12295 
12296 // Shift Left followed by Shift Right.
12297 // This idiom is used by the compiler for the i2b bytecode etc.
12298 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12299 %{
12300   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12301   ins_cost(INSN_COST * 2);
12302   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12303   ins_encode %{
12304     int lshift = $lshift_count$$constant & 63;
12305     int rshift = $rshift_count$$constant & 63;
12306     int s = 63 - lshift;
12307     int r = (rshift - lshift) & 63;
12308     __ ubfm(as_Register($dst$$reg),
12309             as_Register($src$$reg),
12310             r, s);
12311   %}
12312 
12313   ins_pipe(ialu_reg_shift);
12314 %}
12315 
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 
12319 // Shift Left followed by Shift Right.
12320 // This idiom is used by the compiler for the i2b bytecode etc.
12321 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12322 %{
12323   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12324   ins_cost(INSN_COST * 2);
12325   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12326   ins_encode %{
12327     int lshift = $lshift_count$$constant & 31;
12328     int rshift = $rshift_count$$constant & 31;
12329     int s = 31 - lshift;
12330     int r = (rshift - lshift) & 31;
12331     __ ubfmw(as_Register($dst$$reg),
12332             as_Register($src$$reg),
12333             r, s);
12334   %}
12335 
12336   ins_pipe(ialu_reg_shift);
12337 %}
12338 
12339 // Bitfield extract with shift & mask
12340 
12341 // This pattern is automatically generated from aarch64_ad.m4
12342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12343 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12344 %{
12345   match(Set dst (AndI (URShiftI src rshift) mask));
12346   // Make sure we are not going to exceed what ubfxw can do.
12347   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12348 
12349   ins_cost(INSN_COST);
12350   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12351   ins_encode %{
12352     int rshift = $rshift$$constant & 31;
12353     intptr_t mask = $mask$$constant;
12354     int width = exact_log2(mask+1);
12355     __ ubfxw(as_Register($dst$$reg),
12356             as_Register($src$$reg), rshift, width);
12357   %}
12358   ins_pipe(ialu_reg_shift);
12359 %}
12360 
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12364 %{
12365   match(Set dst (AndL (URShiftL src rshift) mask));
12366   // Make sure we are not going to exceed what ubfx can do.
12367   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12368 
12369   ins_cost(INSN_COST);
12370   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12371   ins_encode %{
12372     int rshift = $rshift$$constant & 63;
12373     intptr_t mask = $mask$$constant;
12374     int width = exact_log2_long(mask+1);
12375     __ ubfx(as_Register($dst$$reg),
12376             as_Register($src$$reg), rshift, width);
12377   %}
12378   ins_pipe(ialu_reg_shift);
12379 %}
12380 
12381 
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384 
12385 // We can use ubfx when extending an And with a mask when we know mask
12386 // is positive.  We know that because immI_bitmask guarantees it.
12387 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12388 %{
12389   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12390   // Make sure we are not going to exceed what ubfxw can do.
12391   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12392 
12393   ins_cost(INSN_COST * 2);
12394   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12395   ins_encode %{
12396     int rshift = $rshift$$constant & 31;
12397     intptr_t mask = $mask$$constant;
12398     int width = exact_log2(mask+1);
12399     __ ubfx(as_Register($dst$$reg),
12400             as_Register($src$$reg), rshift, width);
12401   %}
12402   ins_pipe(ialu_reg_shift);
12403 %}
12404 
12405 
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 
12409 // We can use ubfiz when masking by a positive number and then left shifting the result.
12410 // We know that the mask is positive because immI_bitmask guarantees it.
12411 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12412 %{
12413   match(Set dst (LShiftI (AndI src mask) lshift));
12414   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12415 
12416   ins_cost(INSN_COST);
12417   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12418   ins_encode %{
12419     int lshift = $lshift$$constant & 31;
12420     intptr_t mask = $mask$$constant;
12421     int width = exact_log2(mask+1);
12422     __ ubfizw(as_Register($dst$$reg),
12423           as_Register($src$$reg), lshift, width);
12424   %}
12425   ins_pipe(ialu_reg_shift);
12426 %}
12427 
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430 
12431 // We can use ubfiz when masking by a positive number and then left shifting the result.
12432 // We know that the mask is positive because immL_bitmask guarantees it.
12433 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12434 %{
12435   match(Set dst (LShiftL (AndL src mask) lshift));
12436   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12437 
12438   ins_cost(INSN_COST);
12439   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12440   ins_encode %{
12441     int lshift = $lshift$$constant & 63;
12442     intptr_t mask = $mask$$constant;
12443     int width = exact_log2_long(mask+1);
12444     __ ubfiz(as_Register($dst$$reg),
12445           as_Register($src$$reg), lshift, width);
12446   %}
12447   ins_pipe(ialu_reg_shift);
12448 %}
12449 
12450 // This pattern is automatically generated from aarch64_ad.m4
12451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12452 
12453 // We can use ubfiz when masking by a positive number and then left shifting the result.
12454 // We know that the mask is positive because immI_bitmask guarantees it.
12455 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12456 %{
12457   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12458   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12459 
12460   ins_cost(INSN_COST);
12461   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12462   ins_encode %{
12463     int lshift = $lshift$$constant & 31;
12464     intptr_t mask = $mask$$constant;
12465     int width = exact_log2(mask+1);
12466     __ ubfizw(as_Register($dst$$reg),
12467           as_Register($src$$reg), lshift, width);
12468   %}
12469   ins_pipe(ialu_reg_shift);
12470 %}
12471 
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 
12475 // We can use ubfiz when masking by a positive number and then left shifting the result.
12476 // We know that the mask is positive because immL_bitmask guarantees it.
12477 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12478 %{
12479   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12480   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12481 
12482   ins_cost(INSN_COST);
12483   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12484   ins_encode %{
12485     int lshift = $lshift$$constant & 63;
12486     intptr_t mask = $mask$$constant;
12487     int width = exact_log2_long(mask+1);
12488     __ ubfiz(as_Register($dst$$reg),
12489           as_Register($src$$reg), lshift, width);
12490   %}
12491   ins_pipe(ialu_reg_shift);
12492 %}
12493 
12494 
12495 // This pattern is automatically generated from aarch64_ad.m4
12496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12497 
12498 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12499 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12500 %{
12501   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12502   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12503 
12504   ins_cost(INSN_COST);
12505   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12506   ins_encode %{
12507     int lshift = $lshift$$constant & 63;
12508     intptr_t mask = $mask$$constant;
12509     int width = exact_log2(mask+1);
12510     __ ubfiz(as_Register($dst$$reg),
12511              as_Register($src$$reg), lshift, width);
12512   %}
12513   ins_pipe(ialu_reg_shift);
12514 %}
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 
12519 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12520 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12521 %{
12522   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12523   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12524 
12525   ins_cost(INSN_COST);
12526   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12527   ins_encode %{
12528     int lshift = $lshift$$constant & 31;
12529     intptr_t mask = $mask$$constant;
12530     int width = exact_log2(mask+1);
12531     __ ubfiz(as_Register($dst$$reg),
12532              as_Register($src$$reg), lshift, width);
12533   %}
12534   ins_pipe(ialu_reg_shift);
12535 %}
12536 
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539 
12540 // Can skip int2long conversions after AND with small bitmask
12541 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12542 %{
12543   match(Set dst (ConvI2L (AndI src msk)));
12544   ins_cost(INSN_COST);
12545   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12546   ins_encode %{
12547     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12548   %}
12549   ins_pipe(ialu_reg_shift);
12550 %}
12551 
12552 
12553 // Rotations
12554 
12555 // This pattern is automatically generated from aarch64_ad.m4
12556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12557 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12558 %{
12559   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12560   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12561 
12562   ins_cost(INSN_COST);
12563   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12564 
12565   ins_encode %{
12566     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12567             $rshift$$constant & 63);
12568   %}
12569   ins_pipe(ialu_reg_reg_extr);
12570 %}
12571 
12572 
12573 // This pattern is automatically generated from aarch64_ad.m4
12574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12575 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12576 %{
12577   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12578   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12579 
12580   ins_cost(INSN_COST);
12581   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12582 
12583   ins_encode %{
12584     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12585             $rshift$$constant & 31);
12586   %}
12587   ins_pipe(ialu_reg_reg_extr);
12588 %}
12589 
12590 
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12594 %{
12595   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12596   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12597 
12598   ins_cost(INSN_COST);
12599   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12600 
12601   ins_encode %{
12602     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12603             $rshift$$constant & 63);
12604   %}
12605   ins_pipe(ialu_reg_reg_extr);
12606 %}
12607 
12608 
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12612 %{
12613   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12614   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12615 
12616   ins_cost(INSN_COST);
12617   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12618 
12619   ins_encode %{
12620     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12621             $rshift$$constant & 31);
12622   %}
12623   ins_pipe(ialu_reg_reg_extr);
12624 %}
12625 
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12629 %{
12630   match(Set dst (RotateRight src shift));
12631 
12632   ins_cost(INSN_COST);
12633   format %{ "ror    $dst, $src, $shift" %}
12634 
12635   ins_encode %{
12636      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12637                $shift$$constant & 0x1f);
12638   %}
12639   ins_pipe(ialu_reg_reg_vshift);
12640 %}
12641 
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12645 %{
12646   match(Set dst (RotateRight src shift));
12647 
12648   ins_cost(INSN_COST);
12649   format %{ "ror    $dst, $src, $shift" %}
12650 
12651   ins_encode %{
12652      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12653                $shift$$constant & 0x3f);
12654   %}
12655   ins_pipe(ialu_reg_reg_vshift);
12656 %}
12657 
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12661 %{
12662   match(Set dst (RotateRight src shift));
12663 
12664   ins_cost(INSN_COST);
12665   format %{ "ror    $dst, $src, $shift" %}
12666 
12667   ins_encode %{
12668      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12669   %}
12670   ins_pipe(ialu_reg_reg_vshift);
12671 %}
12672 
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12676 %{
12677   match(Set dst (RotateRight src shift));
12678 
12679   ins_cost(INSN_COST);
12680   format %{ "ror    $dst, $src, $shift" %}
12681 
12682   ins_encode %{
12683      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12684   %}
12685   ins_pipe(ialu_reg_reg_vshift);
12686 %}
12687 
12688 // This pattern is automatically generated from aarch64_ad.m4
12689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12690 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12691 %{
12692   match(Set dst (RotateLeft src shift));
12693 
12694   ins_cost(INSN_COST);
12695   format %{ "rol    $dst, $src, $shift" %}
12696 
12697   ins_encode %{
12698      __ subw(rscratch1, zr, as_Register($shift$$reg));
12699      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12700   %}
12701   ins_pipe(ialu_reg_reg_vshift);
12702 %}
12703 
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12707 %{
12708   match(Set dst (RotateLeft src shift));
12709 
12710   ins_cost(INSN_COST);
12711   format %{ "rol    $dst, $src, $shift" %}
12712 
12713   ins_encode %{
12714      __ subw(rscratch1, zr, as_Register($shift$$reg));
12715      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12716   %}
12717   ins_pipe(ialu_reg_reg_vshift);
12718 %}
12719 
12720 
12721 // Add/subtract (extended)
12722 
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12726 %{
12727   match(Set dst (AddL src1 (ConvI2L src2)));
12728   ins_cost(INSN_COST);
12729   format %{ "add  $dst, $src1, $src2, sxtw" %}
12730 
12731    ins_encode %{
12732      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12733             as_Register($src2$$reg), ext::sxtw);
12734    %}
12735   ins_pipe(ialu_reg_reg);
12736 %}
12737 
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12741 %{
12742   match(Set dst (SubL src1 (ConvI2L src2)));
12743   ins_cost(INSN_COST);
12744   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12745 
12746    ins_encode %{
12747      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12748             as_Register($src2$$reg), ext::sxtw);
12749    %}
12750   ins_pipe(ialu_reg_reg);
12751 %}
12752 
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12756 %{
12757   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12758   ins_cost(INSN_COST);
12759   format %{ "add  $dst, $src1, $src2, sxth" %}
12760 
12761    ins_encode %{
12762      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12763             as_Register($src2$$reg), ext::sxth);
12764    %}
12765   ins_pipe(ialu_reg_reg);
12766 %}
12767 
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12771 %{
12772   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12773   ins_cost(INSN_COST);
12774   format %{ "add  $dst, $src1, $src2, sxtb" %}
12775 
12776    ins_encode %{
12777      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12778             as_Register($src2$$reg), ext::sxtb);
12779    %}
12780   ins_pipe(ialu_reg_reg);
12781 %}
12782 
12783 // This pattern is automatically generated from aarch64_ad.m4
12784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12785 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12786 %{
12787   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12788   ins_cost(INSN_COST);
12789   format %{ "add  $dst, $src1, $src2, uxtb" %}
12790 
12791    ins_encode %{
12792      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12793             as_Register($src2$$reg), ext::uxtb);
12794    %}
12795   ins_pipe(ialu_reg_reg);
12796 %}
12797 
12798 // This pattern is automatically generated from aarch64_ad.m4
12799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12800 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12801 %{
12802   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12803   ins_cost(INSN_COST);
12804   format %{ "add  $dst, $src1, $src2, sxth" %}
12805 
12806    ins_encode %{
12807      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12808             as_Register($src2$$reg), ext::sxth);
12809    %}
12810   ins_pipe(ialu_reg_reg);
12811 %}
12812 
12813 // This pattern is automatically generated from aarch64_ad.m4
12814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12815 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12816 %{
12817   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12818   ins_cost(INSN_COST);
12819   format %{ "add  $dst, $src1, $src2, sxtw" %}
12820 
12821    ins_encode %{
12822      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12823             as_Register($src2$$reg), ext::sxtw);
12824    %}
12825   ins_pipe(ialu_reg_reg);
12826 %}
12827 
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12831 %{
12832   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12833   ins_cost(INSN_COST);
12834   format %{ "add  $dst, $src1, $src2, sxtb" %}
12835 
12836    ins_encode %{
12837      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12838             as_Register($src2$$reg), ext::sxtb);
12839    %}
12840   ins_pipe(ialu_reg_reg);
12841 %}
12842 
12843 // This pattern is automatically generated from aarch64_ad.m4
12844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12845 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12846 %{
12847   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12848   ins_cost(INSN_COST);
12849   format %{ "add  $dst, $src1, $src2, uxtb" %}
12850 
12851    ins_encode %{
12852      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12853             as_Register($src2$$reg), ext::uxtb);
12854    %}
12855   ins_pipe(ialu_reg_reg);
12856 %}
12857 
12858 // This pattern is automatically generated from aarch64_ad.m4
12859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12860 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12861 %{
12862   match(Set dst (AddI src1 (AndI src2 mask)));
12863   ins_cost(INSN_COST);
12864   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12865 
12866    ins_encode %{
12867      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12868             as_Register($src2$$reg), ext::uxtb);
12869    %}
12870   ins_pipe(ialu_reg_reg);
12871 %}
12872 
12873 // This pattern is automatically generated from aarch64_ad.m4
12874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12875 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12876 %{
12877   match(Set dst (AddI src1 (AndI src2 mask)));
12878   ins_cost(INSN_COST);
12879   format %{ "addw  $dst, $src1, $src2, uxth" %}
12880 
12881    ins_encode %{
12882      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12883             as_Register($src2$$reg), ext::uxth);
12884    %}
12885   ins_pipe(ialu_reg_reg);
12886 %}
12887 
12888 // This pattern is automatically generated from aarch64_ad.m4
12889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12890 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12891 %{
12892   match(Set dst (AddL src1 (AndL src2 mask)));
12893   ins_cost(INSN_COST);
12894   format %{ "add  $dst, $src1, $src2, uxtb" %}
12895 
12896    ins_encode %{
12897      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12898             as_Register($src2$$reg), ext::uxtb);
12899    %}
12900   ins_pipe(ialu_reg_reg);
12901 %}
12902 
12903 // This pattern is automatically generated from aarch64_ad.m4
12904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12905 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12906 %{
12907   match(Set dst (AddL src1 (AndL src2 mask)));
12908   ins_cost(INSN_COST);
12909   format %{ "add  $dst, $src1, $src2, uxth" %}
12910 
12911    ins_encode %{
12912      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12913             as_Register($src2$$reg), ext::uxth);
12914    %}
12915   ins_pipe(ialu_reg_reg);
12916 %}
12917 
12918 // This pattern is automatically generated from aarch64_ad.m4
12919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12920 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12921 %{
12922   match(Set dst (AddL src1 (AndL src2 mask)));
12923   ins_cost(INSN_COST);
12924   format %{ "add  $dst, $src1, $src2, uxtw" %}
12925 
12926    ins_encode %{
12927      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12928             as_Register($src2$$reg), ext::uxtw);
12929    %}
12930   ins_pipe(ialu_reg_reg);
12931 %}
12932 
12933 // This pattern is automatically generated from aarch64_ad.m4
12934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12935 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12936 %{
12937   match(Set dst (SubI src1 (AndI src2 mask)));
12938   ins_cost(INSN_COST);
12939   format %{ "subw  $dst, $src1, $src2, uxtb" %}
12940 
12941    ins_encode %{
12942      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12943             as_Register($src2$$reg), ext::uxtb);
12944    %}
12945   ins_pipe(ialu_reg_reg);
12946 %}
12947 
12948 // This pattern is automatically generated from aarch64_ad.m4
12949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12950 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12951 %{
12952   match(Set dst (SubI src1 (AndI src2 mask)));
12953   ins_cost(INSN_COST);
12954   format %{ "subw  $dst, $src1, $src2, uxth" %}
12955 
12956    ins_encode %{
12957      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12958             as_Register($src2$$reg), ext::uxth);
12959    %}
12960   ins_pipe(ialu_reg_reg);
12961 %}
12962 
12963 // This pattern is automatically generated from aarch64_ad.m4
12964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12965 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12966 %{
12967   match(Set dst (SubL src1 (AndL src2 mask)));
12968   ins_cost(INSN_COST);
12969   format %{ "sub  $dst, $src1, $src2, uxtb" %}
12970 
12971    ins_encode %{
12972      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12973             as_Register($src2$$reg), ext::uxtb);
12974    %}
12975   ins_pipe(ialu_reg_reg);
12976 %}
12977 
12978 // This pattern is automatically generated from aarch64_ad.m4
12979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12980 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12981 %{
12982   match(Set dst (SubL src1 (AndL src2 mask)));
12983   ins_cost(INSN_COST);
12984   format %{ "sub  $dst, $src1, $src2, uxth" %}
12985 
12986    ins_encode %{
12987      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12988             as_Register($src2$$reg), ext::uxth);
12989    %}
12990   ins_pipe(ialu_reg_reg);
12991 %}
12992 
12993 // This pattern is automatically generated from aarch64_ad.m4
12994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12995 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12996 %{
12997   match(Set dst (SubL src1 (AndL src2 mask)));
12998   ins_cost(INSN_COST);
12999   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13000 
13001    ins_encode %{
13002      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13003             as_Register($src2$$reg), ext::uxtw);
13004    %}
13005   ins_pipe(ialu_reg_reg);
13006 %}
13007 
13008 
13009 // This pattern is automatically generated from aarch64_ad.m4
13010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13011 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13012 %{
13013   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13014   ins_cost(1.9 * INSN_COST);
13015   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13016 
13017    ins_encode %{
13018      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13019             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13020    %}
13021   ins_pipe(ialu_reg_reg_shift);
13022 %}
13023 
13024 // This pattern is automatically generated from aarch64_ad.m4
13025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13026 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13027 %{
13028   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13029   ins_cost(1.9 * INSN_COST);
13030   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13031 
13032    ins_encode %{
13033      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13034             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13035    %}
13036   ins_pipe(ialu_reg_reg_shift);
13037 %}
13038 
13039 // This pattern is automatically generated from aarch64_ad.m4
13040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13041 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13042 %{
13043   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13044   ins_cost(1.9 * INSN_COST);
13045   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13046 
13047    ins_encode %{
13048      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13049             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13050    %}
13051   ins_pipe(ialu_reg_reg_shift);
13052 %}
13053 
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13057 %{
13058   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13059   ins_cost(1.9 * INSN_COST);
13060   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13061 
13062    ins_encode %{
13063      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13064             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13065    %}
13066   ins_pipe(ialu_reg_reg_shift);
13067 %}
13068 
13069 // This pattern is automatically generated from aarch64_ad.m4
13070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13071 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13072 %{
13073   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13074   ins_cost(1.9 * INSN_COST);
13075   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13076 
13077    ins_encode %{
13078      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13079             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13080    %}
13081   ins_pipe(ialu_reg_reg_shift);
13082 %}
13083 
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13087 %{
13088   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13089   ins_cost(1.9 * INSN_COST);
13090   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13091 
13092    ins_encode %{
13093      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13094             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13095    %}
13096   ins_pipe(ialu_reg_reg_shift);
13097 %}
13098 
13099 // This pattern is automatically generated from aarch64_ad.m4
13100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13101 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13102 %{
13103   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13104   ins_cost(1.9 * INSN_COST);
13105   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13106 
13107    ins_encode %{
13108      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13109             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13110    %}
13111   ins_pipe(ialu_reg_reg_shift);
13112 %}
13113 
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13117 %{
13118   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13119   ins_cost(1.9 * INSN_COST);
13120   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13121 
13122    ins_encode %{
13123      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13124             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13125    %}
13126   ins_pipe(ialu_reg_reg_shift);
13127 %}
13128 
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13132 %{
13133   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13134   ins_cost(1.9 * INSN_COST);
13135   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13136 
13137    ins_encode %{
13138      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13139             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13140    %}
13141   ins_pipe(ialu_reg_reg_shift);
13142 %}
13143 
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13147 %{
13148   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13149   ins_cost(1.9 * INSN_COST);
13150   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13151 
13152    ins_encode %{
13153      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13154             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13155    %}
13156   ins_pipe(ialu_reg_reg_shift);
13157 %}
13158 
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13162 %{
13163   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13164   ins_cost(1.9 * INSN_COST);
13165   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13166 
13167    ins_encode %{
13168      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13170    %}
13171   ins_pipe(ialu_reg_reg_shift);
13172 %}
13173 
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13177 %{
13178   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13179   ins_cost(1.9 * INSN_COST);
13180   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13181 
13182    ins_encode %{
13183      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13184             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13185    %}
13186   ins_pipe(ialu_reg_reg_shift);
13187 %}
13188 
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13192 %{
13193   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13194   ins_cost(1.9 * INSN_COST);
13195   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13196 
13197    ins_encode %{
13198      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13199             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13200    %}
13201   ins_pipe(ialu_reg_reg_shift);
13202 %}
13203 
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13207 %{
13208   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13209   ins_cost(1.9 * INSN_COST);
13210   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13211 
13212    ins_encode %{
13213      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13214             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13215    %}
13216   ins_pipe(ialu_reg_reg_shift);
13217 %}
13218 
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13222 %{
13223   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13224   ins_cost(1.9 * INSN_COST);
13225   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13226 
13227    ins_encode %{
13228      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13229             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13230    %}
13231   ins_pipe(ialu_reg_reg_shift);
13232 %}
13233 
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13237 %{
13238   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13239   ins_cost(1.9 * INSN_COST);
13240   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13241 
13242    ins_encode %{
13243      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13244             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13245    %}
13246   ins_pipe(ialu_reg_reg_shift);
13247 %}
13248 
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13252 %{
13253   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13254   ins_cost(1.9 * INSN_COST);
13255   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13256 
13257    ins_encode %{
13258      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13259             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13260    %}
13261   ins_pipe(ialu_reg_reg_shift);
13262 %}
13263 
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13267 %{
13268   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13269   ins_cost(1.9 * INSN_COST);
13270   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13271 
13272    ins_encode %{
13273      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13274             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13275    %}
13276   ins_pipe(ialu_reg_reg_shift);
13277 %}
13278 
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13282 %{
13283   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13284   ins_cost(1.9 * INSN_COST);
13285   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13286 
13287    ins_encode %{
13288      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13289             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13290    %}
13291   ins_pipe(ialu_reg_reg_shift);
13292 %}
13293 
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13297 %{
13298   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13299   ins_cost(1.9 * INSN_COST);
13300   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13301 
13302    ins_encode %{
13303      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13304             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13305    %}
13306   ins_pipe(ialu_reg_reg_shift);
13307 %}
13308 
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13312 %{
13313   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13314   ins_cost(1.9 * INSN_COST);
13315   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13316 
13317    ins_encode %{
13318      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13319             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13320    %}
13321   ins_pipe(ialu_reg_reg_shift);
13322 %}
13323 
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13327 %{
13328   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13329   ins_cost(1.9 * INSN_COST);
13330   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13331 
13332    ins_encode %{
13333      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13334             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13335    %}
13336   ins_pipe(ialu_reg_reg_shift);
13337 %}
13338 
13339 // This pattern is automatically generated from aarch64_ad.m4
13340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13341 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13342 %{
13343   effect(DEF dst, USE src1, USE src2, USE cr);
13344   ins_cost(INSN_COST * 2);
13345   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13346 
13347   ins_encode %{
13348     __ cselw($dst$$Register,
13349              $src1$$Register,
13350              $src2$$Register,
13351              Assembler::LT);
13352   %}
13353   ins_pipe(icond_reg_reg);
13354 %}
13355 
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13359 %{
13360   effect(DEF dst, USE src1, USE src2, USE cr);
13361   ins_cost(INSN_COST * 2);
13362   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13363 
13364   ins_encode %{
13365     __ cselw($dst$$Register,
13366              $src1$$Register,
13367              $src2$$Register,
13368              Assembler::GT);
13369   %}
13370   ins_pipe(icond_reg_reg);
13371 %}
13372 
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13376 %{
13377   effect(DEF dst, USE src1, USE cr);
13378   ins_cost(INSN_COST * 2);
13379   format %{ "cselw $dst, $src1, zr lt\t"  %}
13380 
13381   ins_encode %{
13382     __ cselw($dst$$Register,
13383              $src1$$Register,
13384              zr,
13385              Assembler::LT);
13386   %}
13387   ins_pipe(icond_reg);
13388 %}
13389 
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13393 %{
13394   effect(DEF dst, USE src1, USE cr);
13395   ins_cost(INSN_COST * 2);
13396   format %{ "cselw $dst, $src1, zr gt\t"  %}
13397 
13398   ins_encode %{
13399     __ cselw($dst$$Register,
13400              $src1$$Register,
13401              zr,
13402              Assembler::GT);
13403   %}
13404   ins_pipe(icond_reg);
13405 %}
13406 
13407 // This pattern is automatically generated from aarch64_ad.m4
13408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13409 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13410 %{
13411   effect(DEF dst, USE src1, USE cr);
13412   ins_cost(INSN_COST * 2);
13413   format %{ "csincw $dst, $src1, zr le\t"  %}
13414 
13415   ins_encode %{
13416     __ csincw($dst$$Register,
13417              $src1$$Register,
13418              zr,
13419              Assembler::LE);
13420   %}
13421   ins_pipe(icond_reg);
13422 %}
13423 
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13427 %{
13428   effect(DEF dst, USE src1, USE cr);
13429   ins_cost(INSN_COST * 2);
13430   format %{ "csincw $dst, $src1, zr gt\t"  %}
13431 
13432   ins_encode %{
13433     __ csincw($dst$$Register,
13434              $src1$$Register,
13435              zr,
13436              Assembler::GT);
13437   %}
13438   ins_pipe(icond_reg);
13439 %}
13440 
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13444 %{
13445   effect(DEF dst, USE src1, USE cr);
13446   ins_cost(INSN_COST * 2);
13447   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13448 
13449   ins_encode %{
13450     __ csinvw($dst$$Register,
13451              $src1$$Register,
13452              zr,
13453              Assembler::LT);
13454   %}
13455   ins_pipe(icond_reg);
13456 %}
13457 
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13461 %{
13462   effect(DEF dst, USE src1, USE cr);
13463   ins_cost(INSN_COST * 2);
13464   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13465 
13466   ins_encode %{
13467     __ csinvw($dst$$Register,
13468              $src1$$Register,
13469              zr,
13470              Assembler::GE);
13471   %}
13472   ins_pipe(icond_reg);
13473 %}
13474 
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13478 %{
13479   match(Set dst (MinI src imm));
13480   ins_cost(INSN_COST * 3);
13481   expand %{
13482     rFlagsReg cr;
13483     compI_reg_imm0(cr, src);
13484     cmovI_reg_imm0_lt(dst, src, cr);
13485   %}
13486 %}
13487 
13488 // This pattern is automatically generated from aarch64_ad.m4
13489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13490 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13491 %{
13492   match(Set dst (MinI imm src));
13493   ins_cost(INSN_COST * 3);
13494   expand %{
13495     rFlagsReg cr;
13496     compI_reg_imm0(cr, src);
13497     cmovI_reg_imm0_lt(dst, src, cr);
13498   %}
13499 %}
13500 
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13504 %{
13505   match(Set dst (MinI src imm));
13506   ins_cost(INSN_COST * 3);
13507   expand %{
13508     rFlagsReg cr;
13509     compI_reg_imm0(cr, src);
13510     cmovI_reg_imm1_le(dst, src, cr);
13511   %}
13512 %}
13513 
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13517 %{
13518   match(Set dst (MinI imm src));
13519   ins_cost(INSN_COST * 3);
13520   expand %{
13521     rFlagsReg cr;
13522     compI_reg_imm0(cr, src);
13523     cmovI_reg_imm1_le(dst, src, cr);
13524   %}
13525 %}
13526 
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13530 %{
13531   match(Set dst (MinI src imm));
13532   ins_cost(INSN_COST * 3);
13533   expand %{
13534     rFlagsReg cr;
13535     compI_reg_imm0(cr, src);
13536     cmovI_reg_immM1_lt(dst, src, cr);
13537   %}
13538 %}
13539 
13540 // This pattern is automatically generated from aarch64_ad.m4
13541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13542 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13543 %{
13544   match(Set dst (MinI imm src));
13545   ins_cost(INSN_COST * 3);
13546   expand %{
13547     rFlagsReg cr;
13548     compI_reg_imm0(cr, src);
13549     cmovI_reg_immM1_lt(dst, src, cr);
13550   %}
13551 %}
13552 
13553 // This pattern is automatically generated from aarch64_ad.m4
13554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13555 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13556 %{
13557   match(Set dst (MaxI src imm));
13558   ins_cost(INSN_COST * 3);
13559   expand %{
13560     rFlagsReg cr;
13561     compI_reg_imm0(cr, src);
13562     cmovI_reg_imm0_gt(dst, src, cr);
13563   %}
13564 %}
13565 
13566 // This pattern is automatically generated from aarch64_ad.m4
13567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13568 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13569 %{
13570   match(Set dst (MaxI imm src));
13571   ins_cost(INSN_COST * 3);
13572   expand %{
13573     rFlagsReg cr;
13574     compI_reg_imm0(cr, src);
13575     cmovI_reg_imm0_gt(dst, src, cr);
13576   %}
13577 %}
13578 
13579 // This pattern is automatically generated from aarch64_ad.m4
13580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13581 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13582 %{
13583   match(Set dst (MaxI src imm));
13584   ins_cost(INSN_COST * 3);
13585   expand %{
13586     rFlagsReg cr;
13587     compI_reg_imm0(cr, src);
13588     cmovI_reg_imm1_gt(dst, src, cr);
13589   %}
13590 %}
13591 
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13595 %{
13596   match(Set dst (MaxI imm src));
13597   ins_cost(INSN_COST * 3);
13598   expand %{
13599     rFlagsReg cr;
13600     compI_reg_imm0(cr, src);
13601     cmovI_reg_imm1_gt(dst, src, cr);
13602   %}
13603 %}
13604 
13605 // This pattern is automatically generated from aarch64_ad.m4
13606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13607 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13608 %{
13609   match(Set dst (MaxI src imm));
13610   ins_cost(INSN_COST * 3);
13611   expand %{
13612     rFlagsReg cr;
13613     compI_reg_imm0(cr, src);
13614     cmovI_reg_immM1_ge(dst, src, cr);
13615   %}
13616 %}
13617 
13618 // This pattern is automatically generated from aarch64_ad.m4
13619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13620 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13621 %{
13622   match(Set dst (MaxI imm src));
13623   ins_cost(INSN_COST * 3);
13624   expand %{
13625     rFlagsReg cr;
13626     compI_reg_imm0(cr, src);
13627     cmovI_reg_immM1_ge(dst, src, cr);
13628   %}
13629 %}
13630 
13631 // This pattern is automatically generated from aarch64_ad.m4
13632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13633 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13634 %{
13635   match(Set dst (ReverseI src));
13636   ins_cost(INSN_COST);
13637   format %{ "rbitw  $dst, $src" %}
13638   ins_encode %{
13639     __ rbitw($dst$$Register, $src$$Register);
13640   %}
13641   ins_pipe(ialu_reg);
13642 %}
13643 
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13647 %{
13648   match(Set dst (ReverseL src));
13649   ins_cost(INSN_COST);
13650   format %{ "rbit  $dst, $src" %}
13651   ins_encode %{
13652     __ rbit($dst$$Register, $src$$Register);
13653   %}
13654   ins_pipe(ialu_reg);
13655 %}
13656 
13657 
13658 // END This section of the file is automatically generated. Do not edit --------------
13659 
13660 
13661 // ============================================================================
13662 // Floating Point Arithmetic Instructions
13663 
13664 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13665   match(Set dst (AddF src1 src2));
13666 
13667   ins_cost(INSN_COST * 5);
13668   format %{ "fadds   $dst, $src1, $src2" %}
13669 
13670   ins_encode %{
13671     __ fadds(as_FloatRegister($dst$$reg),
13672              as_FloatRegister($src1$$reg),
13673              as_FloatRegister($src2$$reg));
13674   %}
13675 
13676   ins_pipe(fp_dop_reg_reg_s);
13677 %}
13678 
13679 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13680   match(Set dst (AddD src1 src2));
13681 
13682   ins_cost(INSN_COST * 5);
13683   format %{ "faddd   $dst, $src1, $src2" %}
13684 
13685   ins_encode %{
13686     __ faddd(as_FloatRegister($dst$$reg),
13687              as_FloatRegister($src1$$reg),
13688              as_FloatRegister($src2$$reg));
13689   %}
13690 
13691   ins_pipe(fp_dop_reg_reg_d);
13692 %}
13693 
13694 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13695   match(Set dst (SubF src1 src2));
13696 
13697   ins_cost(INSN_COST * 5);
13698   format %{ "fsubs   $dst, $src1, $src2" %}
13699 
13700   ins_encode %{
13701     __ fsubs(as_FloatRegister($dst$$reg),
13702              as_FloatRegister($src1$$reg),
13703              as_FloatRegister($src2$$reg));
13704   %}
13705 
13706   ins_pipe(fp_dop_reg_reg_s);
13707 %}
13708 
13709 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13710   match(Set dst (SubD src1 src2));
13711 
13712   ins_cost(INSN_COST * 5);
13713   format %{ "fsubd   $dst, $src1, $src2" %}
13714 
13715   ins_encode %{
13716     __ fsubd(as_FloatRegister($dst$$reg),
13717              as_FloatRegister($src1$$reg),
13718              as_FloatRegister($src2$$reg));
13719   %}
13720 
13721   ins_pipe(fp_dop_reg_reg_d);
13722 %}
13723 
13724 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13725   match(Set dst (MulF src1 src2));
13726 
13727   ins_cost(INSN_COST * 6);
13728   format %{ "fmuls   $dst, $src1, $src2" %}
13729 
13730   ins_encode %{
13731     __ fmuls(as_FloatRegister($dst$$reg),
13732              as_FloatRegister($src1$$reg),
13733              as_FloatRegister($src2$$reg));
13734   %}
13735 
13736   ins_pipe(fp_dop_reg_reg_s);
13737 %}
13738 
13739 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13740   match(Set dst (MulD src1 src2));
13741 
13742   ins_cost(INSN_COST * 6);
13743   format %{ "fmuld   $dst, $src1, $src2" %}
13744 
13745   ins_encode %{
13746     __ fmuld(as_FloatRegister($dst$$reg),
13747              as_FloatRegister($src1$$reg),
13748              as_FloatRegister($src2$$reg));
13749   %}
13750 
13751   ins_pipe(fp_dop_reg_reg_d);
13752 %}
13753 
13754 // src1 * src2 + src3
13755 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13756   match(Set dst (FmaF src3 (Binary src1 src2)));
13757 
13758   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13759 
13760   ins_encode %{
13761     assert(UseFMA, "Needs FMA instructions support.");
13762     __ fmadds(as_FloatRegister($dst$$reg),
13763              as_FloatRegister($src1$$reg),
13764              as_FloatRegister($src2$$reg),
13765              as_FloatRegister($src3$$reg));
13766   %}
13767 
13768   ins_pipe(pipe_class_default);
13769 %}
13770 
13771 // src1 * src2 + src3
13772 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13773   match(Set dst (FmaD src3 (Binary src1 src2)));
13774 
13775   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13776 
13777   ins_encode %{
13778     assert(UseFMA, "Needs FMA instructions support.");
13779     __ fmaddd(as_FloatRegister($dst$$reg),
13780              as_FloatRegister($src1$$reg),
13781              as_FloatRegister($src2$$reg),
13782              as_FloatRegister($src3$$reg));
13783   %}
13784 
13785   ins_pipe(pipe_class_default);
13786 %}
13787 
13788 // src1 * (-src2) + src3
13789 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13790 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13791   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13792 
13793   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13794 
13795   ins_encode %{
13796     assert(UseFMA, "Needs FMA instructions support.");
13797     __ fmsubs(as_FloatRegister($dst$$reg),
13798               as_FloatRegister($src1$$reg),
13799               as_FloatRegister($src2$$reg),
13800               as_FloatRegister($src3$$reg));
13801   %}
13802 
13803   ins_pipe(pipe_class_default);
13804 %}
13805 
13806 // src1 * (-src2) + src3
13807 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13808 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13809   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13810 
13811   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13812 
13813   ins_encode %{
13814     assert(UseFMA, "Needs FMA instructions support.");
13815     __ fmsubd(as_FloatRegister($dst$$reg),
13816               as_FloatRegister($src1$$reg),
13817               as_FloatRegister($src2$$reg),
13818               as_FloatRegister($src3$$reg));
13819   %}
13820 
13821   ins_pipe(pipe_class_default);
13822 %}
13823 
13824 // src1 * (-src2) - src3
13825 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13826 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13827   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13828 
13829   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13830 
13831   ins_encode %{
13832     assert(UseFMA, "Needs FMA instructions support.");
13833     __ fnmadds(as_FloatRegister($dst$$reg),
13834                as_FloatRegister($src1$$reg),
13835                as_FloatRegister($src2$$reg),
13836                as_FloatRegister($src3$$reg));
13837   %}
13838 
13839   ins_pipe(pipe_class_default);
13840 %}
13841 
13842 // src1 * (-src2) - src3
13843 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13844 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13845   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13846 
13847   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13848 
13849   ins_encode %{
13850     assert(UseFMA, "Needs FMA instructions support.");
13851     __ fnmaddd(as_FloatRegister($dst$$reg),
13852                as_FloatRegister($src1$$reg),
13853                as_FloatRegister($src2$$reg),
13854                as_FloatRegister($src3$$reg));
13855   %}
13856 
13857   ins_pipe(pipe_class_default);
13858 %}
13859 
13860 // src1 * src2 - src3
13861 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13862   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13863 
13864   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13865 
13866   ins_encode %{
13867     assert(UseFMA, "Needs FMA instructions support.");
13868     __ fnmsubs(as_FloatRegister($dst$$reg),
13869                as_FloatRegister($src1$$reg),
13870                as_FloatRegister($src2$$reg),
13871                as_FloatRegister($src3$$reg));
13872   %}
13873 
13874   ins_pipe(pipe_class_default);
13875 %}
13876 
13877 // src1 * src2 - src3
13878 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13879   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13880 
13881   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13882 
13883   ins_encode %{
13884     assert(UseFMA, "Needs FMA instructions support.");
13885     // n.b. insn name should be fnmsubd
13886     __ fnmsub(as_FloatRegister($dst$$reg),
13887               as_FloatRegister($src1$$reg),
13888               as_FloatRegister($src2$$reg),
13889               as_FloatRegister($src3$$reg));
13890   %}
13891 
13892   ins_pipe(pipe_class_default);
13893 %}
13894 
13895 
13896 // Math.max(FF)F
13897 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13898   match(Set dst (MaxF src1 src2));
13899 
13900   format %{ "fmaxs   $dst, $src1, $src2" %}
13901   ins_encode %{
13902     __ fmaxs(as_FloatRegister($dst$$reg),
13903              as_FloatRegister($src1$$reg),
13904              as_FloatRegister($src2$$reg));
13905   %}
13906 
13907   ins_pipe(fp_dop_reg_reg_s);
13908 %}
13909 
13910 // Math.min(FF)F
13911 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13912   match(Set dst (MinF src1 src2));
13913 
13914   format %{ "fmins   $dst, $src1, $src2" %}
13915   ins_encode %{
13916     __ fmins(as_FloatRegister($dst$$reg),
13917              as_FloatRegister($src1$$reg),
13918              as_FloatRegister($src2$$reg));
13919   %}
13920 
13921   ins_pipe(fp_dop_reg_reg_s);
13922 %}
13923 
13924 // Math.max(DD)D
13925 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13926   match(Set dst (MaxD src1 src2));
13927 
13928   format %{ "fmaxd   $dst, $src1, $src2" %}
13929   ins_encode %{
13930     __ fmaxd(as_FloatRegister($dst$$reg),
13931              as_FloatRegister($src1$$reg),
13932              as_FloatRegister($src2$$reg));
13933   %}
13934 
13935   ins_pipe(fp_dop_reg_reg_d);
13936 %}
13937 
13938 // Math.min(DD)D
13939 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13940   match(Set dst (MinD src1 src2));
13941 
13942   format %{ "fmind   $dst, $src1, $src2" %}
13943   ins_encode %{
13944     __ fmind(as_FloatRegister($dst$$reg),
13945              as_FloatRegister($src1$$reg),
13946              as_FloatRegister($src2$$reg));
13947   %}
13948 
13949   ins_pipe(fp_dop_reg_reg_d);
13950 %}
13951 
13952 
13953 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13954   match(Set dst (DivF src1  src2));
13955 
13956   ins_cost(INSN_COST * 18);
13957   format %{ "fdivs   $dst, $src1, $src2" %}
13958 
13959   ins_encode %{
13960     __ fdivs(as_FloatRegister($dst$$reg),
13961              as_FloatRegister($src1$$reg),
13962              as_FloatRegister($src2$$reg));
13963   %}
13964 
13965   ins_pipe(fp_div_s);
13966 %}
13967 
13968 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13969   match(Set dst (DivD src1  src2));
13970 
13971   ins_cost(INSN_COST * 32);
13972   format %{ "fdivd   $dst, $src1, $src2" %}
13973 
13974   ins_encode %{
13975     __ fdivd(as_FloatRegister($dst$$reg),
13976              as_FloatRegister($src1$$reg),
13977              as_FloatRegister($src2$$reg));
13978   %}
13979 
13980   ins_pipe(fp_div_d);
13981 %}
13982 
13983 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13984   match(Set dst (NegF src));
13985 
13986   ins_cost(INSN_COST * 3);
13987   format %{ "fneg   $dst, $src" %}
13988 
13989   ins_encode %{
13990     __ fnegs(as_FloatRegister($dst$$reg),
13991              as_FloatRegister($src$$reg));
13992   %}
13993 
13994   ins_pipe(fp_uop_s);
13995 %}
13996 
13997 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13998   match(Set dst (NegD src));
13999 
14000   ins_cost(INSN_COST * 3);
14001   format %{ "fnegd   $dst, $src" %}
14002 
14003   ins_encode %{
14004     __ fnegd(as_FloatRegister($dst$$reg),
14005              as_FloatRegister($src$$reg));
14006   %}
14007 
14008   ins_pipe(fp_uop_d);
14009 %}
14010 
14011 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14012 %{
14013   match(Set dst (AbsI src));
14014 
14015   effect(KILL cr);
14016   ins_cost(INSN_COST * 2);
14017   format %{ "cmpw  $src, zr\n\t"
14018             "cnegw $dst, $src, Assembler::LT\t# int abs"
14019   %}
14020 
14021   ins_encode %{
14022     __ cmpw(as_Register($src$$reg), zr);
14023     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14024   %}
14025   ins_pipe(pipe_class_default);
14026 %}
14027 
14028 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14029 %{
14030   match(Set dst (AbsL src));
14031 
14032   effect(KILL cr);
14033   ins_cost(INSN_COST * 2);
14034   format %{ "cmp  $src, zr\n\t"
14035             "cneg $dst, $src, Assembler::LT\t# long abs"
14036   %}
14037 
14038   ins_encode %{
14039     __ cmp(as_Register($src$$reg), zr);
14040     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14041   %}
14042   ins_pipe(pipe_class_default);
14043 %}
14044 
14045 instruct absF_reg(vRegF dst, vRegF src) %{
14046   match(Set dst (AbsF src));
14047 
14048   ins_cost(INSN_COST * 3);
14049   format %{ "fabss   $dst, $src" %}
14050   ins_encode %{
14051     __ fabss(as_FloatRegister($dst$$reg),
14052              as_FloatRegister($src$$reg));
14053   %}
14054 
14055   ins_pipe(fp_uop_s);
14056 %}
14057 
14058 instruct absD_reg(vRegD dst, vRegD src) %{
14059   match(Set dst (AbsD src));
14060 
14061   ins_cost(INSN_COST * 3);
14062   format %{ "fabsd   $dst, $src" %}
14063   ins_encode %{
14064     __ fabsd(as_FloatRegister($dst$$reg),
14065              as_FloatRegister($src$$reg));
14066   %}
14067 
14068   ins_pipe(fp_uop_d);
14069 %}
14070 
14071 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14072   match(Set dst (AbsF (SubF src1 src2)));
14073 
14074   ins_cost(INSN_COST * 3);
14075   format %{ "fabds   $dst, $src1, $src2" %}
14076   ins_encode %{
14077     __ fabds(as_FloatRegister($dst$$reg),
14078              as_FloatRegister($src1$$reg),
14079              as_FloatRegister($src2$$reg));
14080   %}
14081 
14082   ins_pipe(fp_uop_s);
14083 %}
14084 
14085 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14086   match(Set dst (AbsD (SubD src1 src2)));
14087 
14088   ins_cost(INSN_COST * 3);
14089   format %{ "fabdd   $dst, $src1, $src2" %}
14090   ins_encode %{
14091     __ fabdd(as_FloatRegister($dst$$reg),
14092              as_FloatRegister($src1$$reg),
14093              as_FloatRegister($src2$$reg));
14094   %}
14095 
14096   ins_pipe(fp_uop_d);
14097 %}
14098 
14099 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14100   match(Set dst (SqrtD src));
14101 
14102   ins_cost(INSN_COST * 50);
14103   format %{ "fsqrtd  $dst, $src" %}
14104   ins_encode %{
14105     __ fsqrtd(as_FloatRegister($dst$$reg),
14106              as_FloatRegister($src$$reg));
14107   %}
14108 
14109   ins_pipe(fp_div_s);
14110 %}
14111 
14112 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14113   match(Set dst (SqrtF src));
14114 
14115   ins_cost(INSN_COST * 50);
14116   format %{ "fsqrts  $dst, $src" %}
14117   ins_encode %{
14118     __ fsqrts(as_FloatRegister($dst$$reg),
14119              as_FloatRegister($src$$reg));
14120   %}
14121 
14122   ins_pipe(fp_div_d);
14123 %}
14124 
14125 // Math.rint, floor, ceil
14126 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14127   match(Set dst (RoundDoubleMode src rmode));
14128   format %{ "frint  $dst, $src, $rmode" %}
14129   ins_encode %{
14130     switch ($rmode$$constant) {
14131       case RoundDoubleModeNode::rmode_rint:
14132         __ frintnd(as_FloatRegister($dst$$reg),
14133                    as_FloatRegister($src$$reg));
14134         break;
14135       case RoundDoubleModeNode::rmode_floor:
14136         __ frintmd(as_FloatRegister($dst$$reg),
14137                    as_FloatRegister($src$$reg));
14138         break;
14139       case RoundDoubleModeNode::rmode_ceil:
14140         __ frintpd(as_FloatRegister($dst$$reg),
14141                    as_FloatRegister($src$$reg));
14142         break;
14143     }
14144   %}
14145   ins_pipe(fp_uop_d);
14146 %}
14147 
14148 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14149   match(Set dst (CopySignD src1 (Binary src2 zero)));
14150   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14151   format %{ "CopySignD  $dst $src1 $src2" %}
14152   ins_encode %{
14153     FloatRegister dst = as_FloatRegister($dst$$reg),
14154                   src1 = as_FloatRegister($src1$$reg),
14155                   src2 = as_FloatRegister($src2$$reg),
14156                   zero = as_FloatRegister($zero$$reg);
14157     __ fnegd(dst, zero);
14158     __ bsl(dst, __ T8B, src2, src1);
14159   %}
14160   ins_pipe(fp_uop_d);
14161 %}
14162 
14163 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14164   match(Set dst (CopySignF src1 src2));
14165   effect(TEMP_DEF dst, USE src1, USE src2);
14166   format %{ "CopySignF  $dst $src1 $src2" %}
14167   ins_encode %{
14168     FloatRegister dst = as_FloatRegister($dst$$reg),
14169                   src1 = as_FloatRegister($src1$$reg),
14170                   src2 = as_FloatRegister($src2$$reg);
14171     __ movi(dst, __ T2S, 0x80, 24);
14172     __ bsl(dst, __ T8B, src2, src1);
14173   %}
14174   ins_pipe(fp_uop_d);
14175 %}
14176 
14177 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14178   match(Set dst (SignumD src (Binary zero one)));
14179   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14180   format %{ "signumD  $dst, $src" %}
14181   ins_encode %{
14182     FloatRegister src = as_FloatRegister($src$$reg),
14183                   dst = as_FloatRegister($dst$$reg),
14184                   zero = as_FloatRegister($zero$$reg),
14185                   one = as_FloatRegister($one$$reg);
14186     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14187     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14188     // Bit selection instruction gets bit from "one" for each enabled bit in
14189     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14190     // NaN the whole "src" will be copied because "dst" is zero. For all other
14191     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14192     // from "src", and all other bits are copied from 1.0.
14193     __ bsl(dst, __ T8B, one, src);
14194   %}
14195   ins_pipe(fp_uop_d);
14196 %}
14197 
14198 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14199   match(Set dst (SignumF src (Binary zero one)));
14200   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14201   format %{ "signumF  $dst, $src" %}
14202   ins_encode %{
14203     FloatRegister src = as_FloatRegister($src$$reg),
14204                   dst = as_FloatRegister($dst$$reg),
14205                   zero = as_FloatRegister($zero$$reg),
14206                   one = as_FloatRegister($one$$reg);
14207     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14208     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14209     // Bit selection instruction gets bit from "one" for each enabled bit in
14210     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14211     // NaN the whole "src" will be copied because "dst" is zero. For all other
14212     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14213     // from "src", and all other bits are copied from 1.0.
14214     __ bsl(dst, __ T8B, one, src);
14215   %}
14216   ins_pipe(fp_uop_d);
14217 %}
14218 
14219 instruct onspinwait() %{
14220   match(OnSpinWait);
14221   ins_cost(INSN_COST);
14222 
14223   format %{ "onspinwait" %}
14224 
14225   ins_encode %{
14226     __ spin_wait();
14227   %}
14228   ins_pipe(pipe_class_empty);
14229 %}
14230 
14231 // ============================================================================
14232 // Logical Instructions
14233 
14234 // Integer Logical Instructions
14235 
14236 // And Instructions
14237 
14238 
14239 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14240   match(Set dst (AndI src1 src2));
14241 
14242   format %{ "andw  $dst, $src1, $src2\t# int" %}
14243 
14244   ins_cost(INSN_COST);
14245   ins_encode %{
14246     __ andw(as_Register($dst$$reg),
14247             as_Register($src1$$reg),
14248             as_Register($src2$$reg));
14249   %}
14250 
14251   ins_pipe(ialu_reg_reg);
14252 %}
14253 
14254 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14255   match(Set dst (AndI src1 src2));
14256 
14257   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14258 
14259   ins_cost(INSN_COST);
14260   ins_encode %{
14261     __ andw(as_Register($dst$$reg),
14262             as_Register($src1$$reg),
14263             (uint64_t)($src2$$constant));
14264   %}
14265 
14266   ins_pipe(ialu_reg_imm);
14267 %}
14268 
14269 // Or Instructions
14270 
14271 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14272   match(Set dst (OrI src1 src2));
14273 
14274   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14275 
14276   ins_cost(INSN_COST);
14277   ins_encode %{
14278     __ orrw(as_Register($dst$$reg),
14279             as_Register($src1$$reg),
14280             as_Register($src2$$reg));
14281   %}
14282 
14283   ins_pipe(ialu_reg_reg);
14284 %}
14285 
14286 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14287   match(Set dst (OrI src1 src2));
14288 
14289   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14290 
14291   ins_cost(INSN_COST);
14292   ins_encode %{
14293     __ orrw(as_Register($dst$$reg),
14294             as_Register($src1$$reg),
14295             (uint64_t)($src2$$constant));
14296   %}
14297 
14298   ins_pipe(ialu_reg_imm);
14299 %}
14300 
14301 // Xor Instructions
14302 
14303 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14304   match(Set dst (XorI src1 src2));
14305 
14306   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14307 
14308   ins_cost(INSN_COST);
14309   ins_encode %{
14310     __ eorw(as_Register($dst$$reg),
14311             as_Register($src1$$reg),
14312             as_Register($src2$$reg));
14313   %}
14314 
14315   ins_pipe(ialu_reg_reg);
14316 %}
14317 
14318 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14319   match(Set dst (XorI src1 src2));
14320 
14321   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14322 
14323   ins_cost(INSN_COST);
14324   ins_encode %{
14325     __ eorw(as_Register($dst$$reg),
14326             as_Register($src1$$reg),
14327             (uint64_t)($src2$$constant));
14328   %}
14329 
14330   ins_pipe(ialu_reg_imm);
14331 %}
14332 
14333 // Long Logical Instructions
14334 // TODO
14335 
14336 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14337   match(Set dst (AndL src1 src2));
14338 
14339   format %{ "and  $dst, $src1, $src2\t# int" %}
14340 
14341   ins_cost(INSN_COST);
14342   ins_encode %{
14343     __ andr(as_Register($dst$$reg),
14344             as_Register($src1$$reg),
14345             as_Register($src2$$reg));
14346   %}
14347 
14348   ins_pipe(ialu_reg_reg);
14349 %}
14350 
14351 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14352   match(Set dst (AndL src1 src2));
14353 
14354   format %{ "and  $dst, $src1, $src2\t# int" %}
14355 
14356   ins_cost(INSN_COST);
14357   ins_encode %{
14358     __ andr(as_Register($dst$$reg),
14359             as_Register($src1$$reg),
14360             (uint64_t)($src2$$constant));
14361   %}
14362 
14363   ins_pipe(ialu_reg_imm);
14364 %}
14365 
14366 // Or Instructions
14367 
14368 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14369   match(Set dst (OrL src1 src2));
14370 
14371   format %{ "orr  $dst, $src1, $src2\t# int" %}
14372 
14373   ins_cost(INSN_COST);
14374   ins_encode %{
14375     __ orr(as_Register($dst$$reg),
14376            as_Register($src1$$reg),
14377            as_Register($src2$$reg));
14378   %}
14379 
14380   ins_pipe(ialu_reg_reg);
14381 %}
14382 
14383 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14384   match(Set dst (OrL src1 src2));
14385 
14386   format %{ "orr  $dst, $src1, $src2\t# int" %}
14387 
14388   ins_cost(INSN_COST);
14389   ins_encode %{
14390     __ orr(as_Register($dst$$reg),
14391            as_Register($src1$$reg),
14392            (uint64_t)($src2$$constant));
14393   %}
14394 
14395   ins_pipe(ialu_reg_imm);
14396 %}
14397 
14398 // Xor Instructions
14399 
14400 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14401   match(Set dst (XorL src1 src2));
14402 
14403   format %{ "eor  $dst, $src1, $src2\t# int" %}
14404 
14405   ins_cost(INSN_COST);
14406   ins_encode %{
14407     __ eor(as_Register($dst$$reg),
14408            as_Register($src1$$reg),
14409            as_Register($src2$$reg));
14410   %}
14411 
14412   ins_pipe(ialu_reg_reg);
14413 %}
14414 
14415 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14416   match(Set dst (XorL src1 src2));
14417 
14418   ins_cost(INSN_COST);
14419   format %{ "eor  $dst, $src1, $src2\t# int" %}
14420 
14421   ins_encode %{
14422     __ eor(as_Register($dst$$reg),
14423            as_Register($src1$$reg),
14424            (uint64_t)($src2$$constant));
14425   %}
14426 
14427   ins_pipe(ialu_reg_imm);
14428 %}
14429 
14430 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14431 %{
14432   match(Set dst (ConvI2L src));
14433 
14434   ins_cost(INSN_COST);
14435   format %{ "sxtw  $dst, $src\t# i2l" %}
14436   ins_encode %{
14437     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14438   %}
14439   ins_pipe(ialu_reg_shift);
14440 %}
14441 
14442 // this pattern occurs in bigmath arithmetic
14443 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14444 %{
14445   match(Set dst (AndL (ConvI2L src) mask));
14446 
14447   ins_cost(INSN_COST);
14448   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14449   ins_encode %{
14450     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14451   %}
14452 
14453   ins_pipe(ialu_reg_shift);
14454 %}
14455 
14456 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14457   match(Set dst (ConvL2I src));
14458 
14459   ins_cost(INSN_COST);
14460   format %{ "movw  $dst, $src \t// l2i" %}
14461 
14462   ins_encode %{
14463     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14464   %}
14465 
14466   ins_pipe(ialu_reg);
14467 %}
14468 
14469 instruct convD2F_reg(vRegF dst, vRegD src) %{
14470   match(Set dst (ConvD2F src));
14471 
14472   ins_cost(INSN_COST * 5);
14473   format %{ "fcvtd  $dst, $src \t// d2f" %}
14474 
14475   ins_encode %{
14476     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14477   %}
14478 
14479   ins_pipe(fp_d2f);
14480 %}
14481 
14482 instruct convF2D_reg(vRegD dst, vRegF src) %{
14483   match(Set dst (ConvF2D src));
14484 
14485   ins_cost(INSN_COST * 5);
14486   format %{ "fcvts  $dst, $src \t// f2d" %}
14487 
14488   ins_encode %{
14489     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14490   %}
14491 
14492   ins_pipe(fp_f2d);
14493 %}
14494 
14495 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14496   match(Set dst (ConvF2I src));
14497 
14498   ins_cost(INSN_COST * 5);
14499   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14500 
14501   ins_encode %{
14502     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14503   %}
14504 
14505   ins_pipe(fp_f2i);
14506 %}
14507 
14508 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14509   match(Set dst (ConvF2L src));
14510 
14511   ins_cost(INSN_COST * 5);
14512   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14513 
14514   ins_encode %{
14515     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14516   %}
14517 
14518   ins_pipe(fp_f2l);
14519 %}
14520 
14521 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14522   match(Set dst (ConvF2HF src));
14523   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14524             "smov $dst, $tmp\t# move result from $tmp to $dst"
14525   %}
14526   effect(TEMP tmp);
14527   ins_encode %{
14528       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14529   %}
14530   ins_pipe(pipe_slow);
14531 %}
14532 
14533 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14534   match(Set dst (ConvHF2F src));
14535   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14536             "fcvt $dst, $tmp\t# convert half to single precision"
14537   %}
14538   effect(TEMP tmp);
14539   ins_encode %{
14540       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14541   %}
14542   ins_pipe(pipe_slow);
14543 %}
14544 
14545 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14546   match(Set dst (ConvI2F src));
14547 
14548   ins_cost(INSN_COST * 5);
14549   format %{ "scvtfws  $dst, $src \t// i2f" %}
14550 
14551   ins_encode %{
14552     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14553   %}
14554 
14555   ins_pipe(fp_i2f);
14556 %}
14557 
14558 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14559   match(Set dst (ConvL2F src));
14560 
14561   ins_cost(INSN_COST * 5);
14562   format %{ "scvtfs  $dst, $src \t// l2f" %}
14563 
14564   ins_encode %{
14565     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14566   %}
14567 
14568   ins_pipe(fp_l2f);
14569 %}
14570 
14571 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14572   match(Set dst (ConvD2I src));
14573 
14574   ins_cost(INSN_COST * 5);
14575   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14576 
14577   ins_encode %{
14578     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14579   %}
14580 
14581   ins_pipe(fp_d2i);
14582 %}
14583 
14584 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14585   match(Set dst (ConvD2L src));
14586 
14587   ins_cost(INSN_COST * 5);
14588   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14589 
14590   ins_encode %{
14591     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14592   %}
14593 
14594   ins_pipe(fp_d2l);
14595 %}
14596 
14597 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14598   match(Set dst (ConvI2D src));
14599 
14600   ins_cost(INSN_COST * 5);
14601   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14602 
14603   ins_encode %{
14604     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14605   %}
14606 
14607   ins_pipe(fp_i2d);
14608 %}
14609 
14610 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14611   match(Set dst (ConvL2D src));
14612 
14613   ins_cost(INSN_COST * 5);
14614   format %{ "scvtfd  $dst, $src \t// l2d" %}
14615 
14616   ins_encode %{
14617     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14618   %}
14619 
14620   ins_pipe(fp_l2d);
14621 %}
14622 
14623 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14624 %{
14625   match(Set dst (RoundD src));
14626   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14627   format %{ "java_round_double $dst,$src"%}
14628   ins_encode %{
14629     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14630                          as_FloatRegister($ftmp$$reg));
14631   %}
14632   ins_pipe(pipe_slow);
14633 %}
14634 
14635 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14636 %{
14637   match(Set dst (RoundF src));
14638   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14639   format %{ "java_round_float $dst,$src"%}
14640   ins_encode %{
14641     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14642                         as_FloatRegister($ftmp$$reg));
14643   %}
14644   ins_pipe(pipe_slow);
14645 %}
14646 
14647 // stack <-> reg and reg <-> reg shuffles with no conversion
14648 
14649 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14650 
14651   match(Set dst (MoveF2I src));
14652 
14653   effect(DEF dst, USE src);
14654 
14655   ins_cost(4 * INSN_COST);
14656 
14657   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14658 
14659   ins_encode %{
14660     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14661   %}
14662 
14663   ins_pipe(iload_reg_reg);
14664 
14665 %}
14666 
14667 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14668 
14669   match(Set dst (MoveI2F src));
14670 
14671   effect(DEF dst, USE src);
14672 
14673   ins_cost(4 * INSN_COST);
14674 
14675   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14676 
14677   ins_encode %{
14678     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14679   %}
14680 
14681   ins_pipe(pipe_class_memory);
14682 
14683 %}
14684 
14685 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14686 
14687   match(Set dst (MoveD2L src));
14688 
14689   effect(DEF dst, USE src);
14690 
14691   ins_cost(4 * INSN_COST);
14692 
14693   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14694 
14695   ins_encode %{
14696     __ ldr($dst$$Register, Address(sp, $src$$disp));
14697   %}
14698 
14699   ins_pipe(iload_reg_reg);
14700 
14701 %}
14702 
14703 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14704 
14705   match(Set dst (MoveL2D src));
14706 
14707   effect(DEF dst, USE src);
14708 
14709   ins_cost(4 * INSN_COST);
14710 
14711   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14712 
14713   ins_encode %{
14714     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14715   %}
14716 
14717   ins_pipe(pipe_class_memory);
14718 
14719 %}
14720 
14721 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14722 
14723   match(Set dst (MoveF2I src));
14724 
14725   effect(DEF dst, USE src);
14726 
14727   ins_cost(INSN_COST);
14728 
14729   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14730 
14731   ins_encode %{
14732     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14733   %}
14734 
14735   ins_pipe(pipe_class_memory);
14736 
14737 %}
14738 
14739 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14740 
14741   match(Set dst (MoveI2F src));
14742 
14743   effect(DEF dst, USE src);
14744 
14745   ins_cost(INSN_COST);
14746 
14747   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14748 
14749   ins_encode %{
14750     __ strw($src$$Register, Address(sp, $dst$$disp));
14751   %}
14752 
14753   ins_pipe(istore_reg_reg);
14754 
14755 %}
14756 
14757 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14758 
14759   match(Set dst (MoveD2L src));
14760 
14761   effect(DEF dst, USE src);
14762 
14763   ins_cost(INSN_COST);
14764 
14765   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14766 
14767   ins_encode %{
14768     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14769   %}
14770 
14771   ins_pipe(pipe_class_memory);
14772 
14773 %}
14774 
14775 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14776 
14777   match(Set dst (MoveL2D src));
14778 
14779   effect(DEF dst, USE src);
14780 
14781   ins_cost(INSN_COST);
14782 
14783   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14784 
14785   ins_encode %{
14786     __ str($src$$Register, Address(sp, $dst$$disp));
14787   %}
14788 
14789   ins_pipe(istore_reg_reg);
14790 
14791 %}
14792 
14793 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14794 
14795   match(Set dst (MoveF2I src));
14796 
14797   effect(DEF dst, USE src);
14798 
14799   ins_cost(INSN_COST);
14800 
14801   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14802 
14803   ins_encode %{
14804     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14805   %}
14806 
14807   ins_pipe(fp_f2i);
14808 
14809 %}
14810 
14811 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14812 
14813   match(Set dst (MoveI2F src));
14814 
14815   effect(DEF dst, USE src);
14816 
14817   ins_cost(INSN_COST);
14818 
14819   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14820 
14821   ins_encode %{
14822     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14823   %}
14824 
14825   ins_pipe(fp_i2f);
14826 
14827 %}
14828 
14829 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14830 
14831   match(Set dst (MoveD2L src));
14832 
14833   effect(DEF dst, USE src);
14834 
14835   ins_cost(INSN_COST);
14836 
14837   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14838 
14839   ins_encode %{
14840     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14841   %}
14842 
14843   ins_pipe(fp_d2l);
14844 
14845 %}
14846 
14847 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14848 
14849   match(Set dst (MoveL2D src));
14850 
14851   effect(DEF dst, USE src);
14852 
14853   ins_cost(INSN_COST);
14854 
14855   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14856 
14857   ins_encode %{
14858     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14859   %}
14860 
14861   ins_pipe(fp_l2d);
14862 
14863 %}
14864 
14865 // ============================================================================
14866 // clearing of an array
14867 
14868 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14869 %{
14870   match(Set dummy (ClearArray (Binary cnt base) zero));
14871   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14872 
14873   ins_cost(4 * INSN_COST);
14874   format %{ "ClearArray $cnt, $base" %}
14875 
14876   ins_encode %{
14877     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14878     if (tpc == nullptr) {
14879       ciEnv::current()->record_failure("CodeCache is full");
14880       return;
14881     }
14882   %}
14883 
14884   ins_pipe(pipe_class_memory);
14885 %}
14886 
14887 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14888 %{
14889   predicate(((ClearArrayNode*)n)->word_copy_only());
14890   match(Set dummy (ClearArray (Binary cnt base) val));
14891   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14892 
14893   ins_cost(4 * INSN_COST);
14894   format %{ "ClearArray $cnt, $base, $val" %}
14895 
14896   ins_encode %{
14897     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14898   %}
14899 
14900   ins_pipe(pipe_class_memory);
14901 %}
14902 
14903 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14904 %{
14905   predicate((uint64_t)n->in(2)->get_long()
14906             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14907             && !((ClearArrayNode*)n)->word_copy_only());
14908   match(Set dummy (ClearArray cnt base));
14909   effect(TEMP temp, USE_KILL base, KILL cr);
14910 
14911   ins_cost(4 * INSN_COST);
14912   format %{ "ClearArray $cnt, $base" %}
14913 
14914   ins_encode %{
14915     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14916     if (tpc == nullptr) {
14917       ciEnv::current()->record_failure("CodeCache is full");
14918       return;
14919     }
14920   %}
14921 
14922   ins_pipe(pipe_class_memory);
14923 %}
14924 
14925 // ============================================================================
14926 // Overflow Math Instructions
14927 
14928 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14929 %{
14930   match(Set cr (OverflowAddI op1 op2));
14931 
14932   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14933   ins_cost(INSN_COST);
14934   ins_encode %{
14935     __ cmnw($op1$$Register, $op2$$Register);
14936   %}
14937 
14938   ins_pipe(icmp_reg_reg);
14939 %}
14940 
14941 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14942 %{
14943   match(Set cr (OverflowAddI op1 op2));
14944 
14945   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14946   ins_cost(INSN_COST);
14947   ins_encode %{
14948     __ cmnw($op1$$Register, $op2$$constant);
14949   %}
14950 
14951   ins_pipe(icmp_reg_imm);
14952 %}
14953 
14954 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14955 %{
14956   match(Set cr (OverflowAddL op1 op2));
14957 
14958   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14959   ins_cost(INSN_COST);
14960   ins_encode %{
14961     __ cmn($op1$$Register, $op2$$Register);
14962   %}
14963 
14964   ins_pipe(icmp_reg_reg);
14965 %}
14966 
14967 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14968 %{
14969   match(Set cr (OverflowAddL op1 op2));
14970 
14971   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
14972   ins_cost(INSN_COST);
14973   ins_encode %{
14974     __ adds(zr, $op1$$Register, $op2$$constant);
14975   %}
14976 
14977   ins_pipe(icmp_reg_imm);
14978 %}
14979 
14980 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14981 %{
14982   match(Set cr (OverflowSubI op1 op2));
14983 
14984   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14985   ins_cost(INSN_COST);
14986   ins_encode %{
14987     __ cmpw($op1$$Register, $op2$$Register);
14988   %}
14989 
14990   ins_pipe(icmp_reg_reg);
14991 %}
14992 
14993 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14994 %{
14995   match(Set cr (OverflowSubI op1 op2));
14996 
14997   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14998   ins_cost(INSN_COST);
14999   ins_encode %{
15000     __ cmpw($op1$$Register, $op2$$constant);
15001   %}
15002 
15003   ins_pipe(icmp_reg_imm);
15004 %}
15005 
15006 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15007 %{
15008   match(Set cr (OverflowSubL op1 op2));
15009 
15010   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15011   ins_cost(INSN_COST);
15012   ins_encode %{
15013     __ cmp($op1$$Register, $op2$$Register);
15014   %}
15015 
15016   ins_pipe(icmp_reg_reg);
15017 %}
15018 
15019 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15020 %{
15021   match(Set cr (OverflowSubL op1 op2));
15022 
15023   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15024   ins_cost(INSN_COST);
15025   ins_encode %{
15026     __ subs(zr, $op1$$Register, $op2$$constant);
15027   %}
15028 
15029   ins_pipe(icmp_reg_imm);
15030 %}
15031 
15032 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15033 %{
15034   match(Set cr (OverflowSubI zero op1));
15035 
15036   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15037   ins_cost(INSN_COST);
15038   ins_encode %{
15039     __ cmpw(zr, $op1$$Register);
15040   %}
15041 
15042   ins_pipe(icmp_reg_imm);
15043 %}
15044 
15045 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15046 %{
15047   match(Set cr (OverflowSubL zero op1));
15048 
15049   format %{ "cmp   zr, $op1\t# overflow check long" %}
15050   ins_cost(INSN_COST);
15051   ins_encode %{
15052     __ cmp(zr, $op1$$Register);
15053   %}
15054 
15055   ins_pipe(icmp_reg_imm);
15056 %}
15057 
15058 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15059 %{
15060   match(Set cr (OverflowMulI op1 op2));
15061 
15062   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15063             "cmp   rscratch1, rscratch1, sxtw\n\t"
15064             "movw  rscratch1, #0x80000000\n\t"
15065             "cselw rscratch1, rscratch1, zr, NE\n\t"
15066             "cmpw  rscratch1, #1" %}
15067   ins_cost(5 * INSN_COST);
15068   ins_encode %{
15069     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15070     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15071     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15072     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15073     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15074   %}
15075 
15076   ins_pipe(pipe_slow);
15077 %}
15078 
15079 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15080 %{
15081   match(If cmp (OverflowMulI op1 op2));
15082   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15083             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15084   effect(USE labl, KILL cr);
15085 
15086   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15087             "cmp   rscratch1, rscratch1, sxtw\n\t"
15088             "b$cmp   $labl" %}
15089   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15090   ins_encode %{
15091     Label* L = $labl$$label;
15092     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15093     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15094     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15095     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15096   %}
15097 
15098   ins_pipe(pipe_serial);
15099 %}
15100 
15101 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15102 %{
15103   match(Set cr (OverflowMulL op1 op2));
15104 
15105   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15106             "smulh rscratch2, $op1, $op2\n\t"
15107             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15108             "movw  rscratch1, #0x80000000\n\t"
15109             "cselw rscratch1, rscratch1, zr, NE\n\t"
15110             "cmpw  rscratch1, #1" %}
15111   ins_cost(6 * INSN_COST);
15112   ins_encode %{
15113     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15114     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15115     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15116     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15117     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15118     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15119   %}
15120 
15121   ins_pipe(pipe_slow);
15122 %}
15123 
15124 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15125 %{
15126   match(If cmp (OverflowMulL op1 op2));
15127   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15128             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15129   effect(USE labl, KILL cr);
15130 
15131   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15132             "smulh rscratch2, $op1, $op2\n\t"
15133             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15134             "b$cmp $labl" %}
15135   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15136   ins_encode %{
15137     Label* L = $labl$$label;
15138     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15139     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15140     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15141     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15142     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15143   %}
15144 
15145   ins_pipe(pipe_serial);
15146 %}
15147 
15148 // ============================================================================
15149 // Compare Instructions
15150 
15151 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15152 %{
15153   match(Set cr (CmpI op1 op2));
15154 
15155   effect(DEF cr, USE op1, USE op2);
15156 
15157   ins_cost(INSN_COST);
15158   format %{ "cmpw  $op1, $op2" %}
15159 
15160   ins_encode(aarch64_enc_cmpw(op1, op2));
15161 
15162   ins_pipe(icmp_reg_reg);
15163 %}
15164 
15165 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15166 %{
15167   match(Set cr (CmpI op1 zero));
15168 
15169   effect(DEF cr, USE op1);
15170 
15171   ins_cost(INSN_COST);
15172   format %{ "cmpw $op1, 0" %}
15173 
15174   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15175 
15176   ins_pipe(icmp_reg_imm);
15177 %}
15178 
15179 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15180 %{
15181   match(Set cr (CmpI op1 op2));
15182 
15183   effect(DEF cr, USE op1);
15184 
15185   ins_cost(INSN_COST);
15186   format %{ "cmpw  $op1, $op2" %}
15187 
15188   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15189 
15190   ins_pipe(icmp_reg_imm);
15191 %}
15192 
15193 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15194 %{
15195   match(Set cr (CmpI op1 op2));
15196 
15197   effect(DEF cr, USE op1);
15198 
15199   ins_cost(INSN_COST * 2);
15200   format %{ "cmpw  $op1, $op2" %}
15201 
15202   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15203 
15204   ins_pipe(icmp_reg_imm);
15205 %}
15206 
15207 // Unsigned compare Instructions; really, same as signed compare
15208 // except it should only be used to feed an If or a CMovI which takes a
15209 // cmpOpU.
15210 
15211 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15212 %{
15213   match(Set cr (CmpU op1 op2));
15214 
15215   effect(DEF cr, USE op1, USE op2);
15216 
15217   ins_cost(INSN_COST);
15218   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15219 
15220   ins_encode(aarch64_enc_cmpw(op1, op2));
15221 
15222   ins_pipe(icmp_reg_reg);
15223 %}
15224 
15225 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15226 %{
15227   match(Set cr (CmpU op1 zero));
15228 
15229   effect(DEF cr, USE op1);
15230 
15231   ins_cost(INSN_COST);
15232   format %{ "cmpw $op1, #0\t# unsigned" %}
15233 
15234   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15235 
15236   ins_pipe(icmp_reg_imm);
15237 %}
15238 
15239 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15240 %{
15241   match(Set cr (CmpU op1 op2));
15242 
15243   effect(DEF cr, USE op1);
15244 
15245   ins_cost(INSN_COST);
15246   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15247 
15248   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15249 
15250   ins_pipe(icmp_reg_imm);
15251 %}
15252 
15253 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15254 %{
15255   match(Set cr (CmpU op1 op2));
15256 
15257   effect(DEF cr, USE op1);
15258 
15259   ins_cost(INSN_COST * 2);
15260   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15261 
15262   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15263 
15264   ins_pipe(icmp_reg_imm);
15265 %}
15266 
15267 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15268 %{
15269   match(Set cr (CmpL op1 op2));
15270 
15271   effect(DEF cr, USE op1, USE op2);
15272 
15273   ins_cost(INSN_COST);
15274   format %{ "cmp  $op1, $op2" %}
15275 
15276   ins_encode(aarch64_enc_cmp(op1, op2));
15277 
15278   ins_pipe(icmp_reg_reg);
15279 %}
15280 
15281 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15282 %{
15283   match(Set cr (CmpL op1 zero));
15284 
15285   effect(DEF cr, USE op1);
15286 
15287   ins_cost(INSN_COST);
15288   format %{ "tst  $op1" %}
15289 
15290   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15291 
15292   ins_pipe(icmp_reg_imm);
15293 %}
15294 
15295 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15296 %{
15297   match(Set cr (CmpL op1 op2));
15298 
15299   effect(DEF cr, USE op1);
15300 
15301   ins_cost(INSN_COST);
15302   format %{ "cmp  $op1, $op2" %}
15303 
15304   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15305 
15306   ins_pipe(icmp_reg_imm);
15307 %}
15308 
15309 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15310 %{
15311   match(Set cr (CmpL op1 op2));
15312 
15313   effect(DEF cr, USE op1);
15314 
15315   ins_cost(INSN_COST * 2);
15316   format %{ "cmp  $op1, $op2" %}
15317 
15318   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15319 
15320   ins_pipe(icmp_reg_imm);
15321 %}
15322 
15323 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15324 %{
15325   match(Set cr (CmpUL op1 op2));
15326 
15327   effect(DEF cr, USE op1, USE op2);
15328 
15329   ins_cost(INSN_COST);
15330   format %{ "cmp  $op1, $op2" %}
15331 
15332   ins_encode(aarch64_enc_cmp(op1, op2));
15333 
15334   ins_pipe(icmp_reg_reg);
15335 %}
15336 
15337 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15338 %{
15339   match(Set cr (CmpUL op1 zero));
15340 
15341   effect(DEF cr, USE op1);
15342 
15343   ins_cost(INSN_COST);
15344   format %{ "tst  $op1" %}
15345 
15346   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15347 
15348   ins_pipe(icmp_reg_imm);
15349 %}
15350 
15351 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15352 %{
15353   match(Set cr (CmpUL op1 op2));
15354 
15355   effect(DEF cr, USE op1);
15356 
15357   ins_cost(INSN_COST);
15358   format %{ "cmp  $op1, $op2" %}
15359 
15360   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15361 
15362   ins_pipe(icmp_reg_imm);
15363 %}
15364 
15365 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15366 %{
15367   match(Set cr (CmpUL op1 op2));
15368 
15369   effect(DEF cr, USE op1);
15370 
15371   ins_cost(INSN_COST * 2);
15372   format %{ "cmp  $op1, $op2" %}
15373 
15374   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15375 
15376   ins_pipe(icmp_reg_imm);
15377 %}
15378 
15379 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15380 %{
15381   match(Set cr (CmpP op1 op2));
15382 
15383   effect(DEF cr, USE op1, USE op2);
15384 
15385   ins_cost(INSN_COST);
15386   format %{ "cmp  $op1, $op2\t // ptr" %}
15387 
15388   ins_encode(aarch64_enc_cmpp(op1, op2));
15389 
15390   ins_pipe(icmp_reg_reg);
15391 %}
15392 
15393 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15394 %{
15395   match(Set cr (CmpN op1 op2));
15396 
15397   effect(DEF cr, USE op1, USE op2);
15398 
15399   ins_cost(INSN_COST);
15400   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15401 
15402   ins_encode(aarch64_enc_cmpn(op1, op2));
15403 
15404   ins_pipe(icmp_reg_reg);
15405 %}
15406 
15407 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15408 %{
15409   match(Set cr (CmpP op1 zero));
15410 
15411   effect(DEF cr, USE op1, USE zero);
15412 
15413   ins_cost(INSN_COST);
15414   format %{ "cmp  $op1, 0\t // ptr" %}
15415 
15416   ins_encode(aarch64_enc_testp(op1));
15417 
15418   ins_pipe(icmp_reg_imm);
15419 %}
15420 
15421 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15422 %{
15423   match(Set cr (CmpN op1 zero));
15424 
15425   effect(DEF cr, USE op1, USE zero);
15426 
15427   ins_cost(INSN_COST);
15428   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15429 
15430   ins_encode(aarch64_enc_testn(op1));
15431 
15432   ins_pipe(icmp_reg_imm);
15433 %}
15434 
15435 // FP comparisons
15436 //
15437 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15438 // using normal cmpOp. See declaration of rFlagsReg for details.
15439 
15440 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15441 %{
15442   match(Set cr (CmpF src1 src2));
15443 
15444   ins_cost(3 * INSN_COST);
15445   format %{ "fcmps $src1, $src2" %}
15446 
15447   ins_encode %{
15448     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15449   %}
15450 
15451   ins_pipe(pipe_class_compare);
15452 %}
15453 
15454 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15455 %{
15456   match(Set cr (CmpF src1 src2));
15457 
15458   ins_cost(3 * INSN_COST);
15459   format %{ "fcmps $src1, 0.0" %}
15460 
15461   ins_encode %{
15462     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15463   %}
15464 
15465   ins_pipe(pipe_class_compare);
15466 %}
15467 // FROM HERE
15468 
15469 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15470 %{
15471   match(Set cr (CmpD src1 src2));
15472 
15473   ins_cost(3 * INSN_COST);
15474   format %{ "fcmpd $src1, $src2" %}
15475 
15476   ins_encode %{
15477     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15478   %}
15479 
15480   ins_pipe(pipe_class_compare);
15481 %}
15482 
15483 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15484 %{
15485   match(Set cr (CmpD src1 src2));
15486 
15487   ins_cost(3 * INSN_COST);
15488   format %{ "fcmpd $src1, 0.0" %}
15489 
15490   ins_encode %{
15491     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15492   %}
15493 
15494   ins_pipe(pipe_class_compare);
15495 %}
15496 
15497 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15498 %{
15499   match(Set dst (CmpF3 src1 src2));
15500   effect(KILL cr);
15501 
15502   ins_cost(5 * INSN_COST);
15503   format %{ "fcmps $src1, $src2\n\t"
15504             "csinvw($dst, zr, zr, eq\n\t"
15505             "csnegw($dst, $dst, $dst, lt)"
15506   %}
15507 
15508   ins_encode %{
15509     Label done;
15510     FloatRegister s1 = as_FloatRegister($src1$$reg);
15511     FloatRegister s2 = as_FloatRegister($src2$$reg);
15512     Register d = as_Register($dst$$reg);
15513     __ fcmps(s1, s2);
15514     // installs 0 if EQ else -1
15515     __ csinvw(d, zr, zr, Assembler::EQ);
15516     // keeps -1 if less or unordered else installs 1
15517     __ csnegw(d, d, d, Assembler::LT);
15518     __ bind(done);
15519   %}
15520 
15521   ins_pipe(pipe_class_default);
15522 
15523 %}
15524 
15525 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15526 %{
15527   match(Set dst (CmpD3 src1 src2));
15528   effect(KILL cr);
15529 
15530   ins_cost(5 * INSN_COST);
15531   format %{ "fcmpd $src1, $src2\n\t"
15532             "csinvw($dst, zr, zr, eq\n\t"
15533             "csnegw($dst, $dst, $dst, lt)"
15534   %}
15535 
15536   ins_encode %{
15537     Label done;
15538     FloatRegister s1 = as_FloatRegister($src1$$reg);
15539     FloatRegister s2 = as_FloatRegister($src2$$reg);
15540     Register d = as_Register($dst$$reg);
15541     __ fcmpd(s1, s2);
15542     // installs 0 if EQ else -1
15543     __ csinvw(d, zr, zr, Assembler::EQ);
15544     // keeps -1 if less or unordered else installs 1
15545     __ csnegw(d, d, d, Assembler::LT);
15546     __ bind(done);
15547   %}
15548   ins_pipe(pipe_class_default);
15549 
15550 %}
15551 
15552 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15553 %{
15554   match(Set dst (CmpF3 src1 zero));
15555   effect(KILL cr);
15556 
15557   ins_cost(5 * INSN_COST);
15558   format %{ "fcmps $src1, 0.0\n\t"
15559             "csinvw($dst, zr, zr, eq\n\t"
15560             "csnegw($dst, $dst, $dst, lt)"
15561   %}
15562 
15563   ins_encode %{
15564     Label done;
15565     FloatRegister s1 = as_FloatRegister($src1$$reg);
15566     Register d = as_Register($dst$$reg);
15567     __ fcmps(s1, 0.0);
15568     // installs 0 if EQ else -1
15569     __ csinvw(d, zr, zr, Assembler::EQ);
15570     // keeps -1 if less or unordered else installs 1
15571     __ csnegw(d, d, d, Assembler::LT);
15572     __ bind(done);
15573   %}
15574 
15575   ins_pipe(pipe_class_default);
15576 
15577 %}
15578 
15579 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15580 %{
15581   match(Set dst (CmpD3 src1 zero));
15582   effect(KILL cr);
15583 
15584   ins_cost(5 * INSN_COST);
15585   format %{ "fcmpd $src1, 0.0\n\t"
15586             "csinvw($dst, zr, zr, eq\n\t"
15587             "csnegw($dst, $dst, $dst, lt)"
15588   %}
15589 
15590   ins_encode %{
15591     Label done;
15592     FloatRegister s1 = as_FloatRegister($src1$$reg);
15593     Register d = as_Register($dst$$reg);
15594     __ fcmpd(s1, 0.0);
15595     // installs 0 if EQ else -1
15596     __ csinvw(d, zr, zr, Assembler::EQ);
15597     // keeps -1 if less or unordered else installs 1
15598     __ csnegw(d, d, d, Assembler::LT);
15599     __ bind(done);
15600   %}
15601   ins_pipe(pipe_class_default);
15602 
15603 %}
15604 
15605 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15606 %{
15607   match(Set dst (CmpLTMask p q));
15608   effect(KILL cr);
15609 
15610   ins_cost(3 * INSN_COST);
15611 
15612   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15613             "csetw $dst, lt\n\t"
15614             "subw $dst, zr, $dst"
15615   %}
15616 
15617   ins_encode %{
15618     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15619     __ csetw(as_Register($dst$$reg), Assembler::LT);
15620     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15621   %}
15622 
15623   ins_pipe(ialu_reg_reg);
15624 %}
15625 
15626 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15627 %{
15628   match(Set dst (CmpLTMask src zero));
15629   effect(KILL cr);
15630 
15631   ins_cost(INSN_COST);
15632 
15633   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15634 
15635   ins_encode %{
15636     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15637   %}
15638 
15639   ins_pipe(ialu_reg_shift);
15640 %}
15641 
15642 // ============================================================================
15643 // Max and Min
15644 
15645 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15646 
15647 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15648 %{
15649   effect(DEF cr, USE src);
15650   ins_cost(INSN_COST);
15651   format %{ "cmpw $src, 0" %}
15652 
15653   ins_encode %{
15654     __ cmpw($src$$Register, 0);
15655   %}
15656   ins_pipe(icmp_reg_imm);
15657 %}
15658 
15659 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15660 %{
15661   match(Set dst (MinI src1 src2));
15662   ins_cost(INSN_COST * 3);
15663 
15664   expand %{
15665     rFlagsReg cr;
15666     compI_reg_reg(cr, src1, src2);
15667     cmovI_reg_reg_lt(dst, src1, src2, cr);
15668   %}
15669 %}
15670 
15671 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15672 %{
15673   match(Set dst (MaxI src1 src2));
15674   ins_cost(INSN_COST * 3);
15675 
15676   expand %{
15677     rFlagsReg cr;
15678     compI_reg_reg(cr, src1, src2);
15679     cmovI_reg_reg_gt(dst, src1, src2, cr);
15680   %}
15681 %}
15682 
15683 
15684 // ============================================================================
15685 // Branch Instructions
15686 
15687 // Direct Branch.
15688 instruct branch(label lbl)
15689 %{
15690   match(Goto);
15691 
15692   effect(USE lbl);
15693 
15694   ins_cost(BRANCH_COST);
15695   format %{ "b  $lbl" %}
15696 
15697   ins_encode(aarch64_enc_b(lbl));
15698 
15699   ins_pipe(pipe_branch);
15700 %}
15701 
15702 // Conditional Near Branch
15703 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15704 %{
15705   // Same match rule as `branchConFar'.
15706   match(If cmp cr);
15707 
15708   effect(USE lbl);
15709 
15710   ins_cost(BRANCH_COST);
15711   // If set to 1 this indicates that the current instruction is a
15712   // short variant of a long branch. This avoids using this
15713   // instruction in first-pass matching. It will then only be used in
15714   // the `Shorten_branches' pass.
15715   // ins_short_branch(1);
15716   format %{ "b$cmp  $lbl" %}
15717 
15718   ins_encode(aarch64_enc_br_con(cmp, lbl));
15719 
15720   ins_pipe(pipe_branch_cond);
15721 %}
15722 
15723 // Conditional Near Branch Unsigned
15724 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15725 %{
15726   // Same match rule as `branchConFar'.
15727   match(If cmp cr);
15728 
15729   effect(USE lbl);
15730 
15731   ins_cost(BRANCH_COST);
15732   // If set to 1 this indicates that the current instruction is a
15733   // short variant of a long branch. This avoids using this
15734   // instruction in first-pass matching. It will then only be used in
15735   // the `Shorten_branches' pass.
15736   // ins_short_branch(1);
15737   format %{ "b$cmp  $lbl\t# unsigned" %}
15738 
15739   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15740 
15741   ins_pipe(pipe_branch_cond);
15742 %}
15743 
15744 // Make use of CBZ and CBNZ.  These instructions, as well as being
15745 // shorter than (cmp; branch), have the additional benefit of not
15746 // killing the flags.
15747 
15748 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15749   match(If cmp (CmpI op1 op2));
15750   effect(USE labl);
15751 
15752   ins_cost(BRANCH_COST);
15753   format %{ "cbw$cmp   $op1, $labl" %}
15754   ins_encode %{
15755     Label* L = $labl$$label;
15756     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15757     if (cond == Assembler::EQ)
15758       __ cbzw($op1$$Register, *L);
15759     else
15760       __ cbnzw($op1$$Register, *L);
15761   %}
15762   ins_pipe(pipe_cmp_branch);
15763 %}
15764 
15765 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15766   match(If cmp (CmpL op1 op2));
15767   effect(USE labl);
15768 
15769   ins_cost(BRANCH_COST);
15770   format %{ "cb$cmp   $op1, $labl" %}
15771   ins_encode %{
15772     Label* L = $labl$$label;
15773     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15774     if (cond == Assembler::EQ)
15775       __ cbz($op1$$Register, *L);
15776     else
15777       __ cbnz($op1$$Register, *L);
15778   %}
15779   ins_pipe(pipe_cmp_branch);
15780 %}
15781 
15782 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15783   match(If cmp (CmpP op1 op2));
15784   effect(USE labl);
15785 
15786   ins_cost(BRANCH_COST);
15787   format %{ "cb$cmp   $op1, $labl" %}
15788   ins_encode %{
15789     Label* L = $labl$$label;
15790     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15791     if (cond == Assembler::EQ)
15792       __ cbz($op1$$Register, *L);
15793     else
15794       __ cbnz($op1$$Register, *L);
15795   %}
15796   ins_pipe(pipe_cmp_branch);
15797 %}
15798 
15799 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15800   match(If cmp (CmpN op1 op2));
15801   effect(USE labl);
15802 
15803   ins_cost(BRANCH_COST);
15804   format %{ "cbw$cmp   $op1, $labl" %}
15805   ins_encode %{
15806     Label* L = $labl$$label;
15807     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15808     if (cond == Assembler::EQ)
15809       __ cbzw($op1$$Register, *L);
15810     else
15811       __ cbnzw($op1$$Register, *L);
15812   %}
15813   ins_pipe(pipe_cmp_branch);
15814 %}
15815 
15816 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15817   match(If cmp (CmpP (DecodeN oop) zero));
15818   effect(USE labl);
15819 
15820   ins_cost(BRANCH_COST);
15821   format %{ "cb$cmp   $oop, $labl" %}
15822   ins_encode %{
15823     Label* L = $labl$$label;
15824     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15825     if (cond == Assembler::EQ)
15826       __ cbzw($oop$$Register, *L);
15827     else
15828       __ cbnzw($oop$$Register, *L);
15829   %}
15830   ins_pipe(pipe_cmp_branch);
15831 %}
15832 
15833 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15834   match(If cmp (CmpU op1 op2));
15835   effect(USE labl);
15836 
15837   ins_cost(BRANCH_COST);
15838   format %{ "cbw$cmp   $op1, $labl" %}
15839   ins_encode %{
15840     Label* L = $labl$$label;
15841     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15842     if (cond == Assembler::EQ || cond == Assembler::LS) {
15843       __ cbzw($op1$$Register, *L);
15844     } else {
15845       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15846       __ cbnzw($op1$$Register, *L);
15847     }
15848   %}
15849   ins_pipe(pipe_cmp_branch);
15850 %}
15851 
15852 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15853   match(If cmp (CmpUL op1 op2));
15854   effect(USE labl);
15855 
15856   ins_cost(BRANCH_COST);
15857   format %{ "cb$cmp   $op1, $labl" %}
15858   ins_encode %{
15859     Label* L = $labl$$label;
15860     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15861     if (cond == Assembler::EQ || cond == Assembler::LS) {
15862       __ cbz($op1$$Register, *L);
15863     } else {
15864       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15865       __ cbnz($op1$$Register, *L);
15866     }
15867   %}
15868   ins_pipe(pipe_cmp_branch);
15869 %}
15870 
15871 // Test bit and Branch
15872 
15873 // Patterns for short (< 32KiB) variants
15874 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15875   match(If cmp (CmpL op1 op2));
15876   effect(USE labl);
15877 
15878   ins_cost(BRANCH_COST);
15879   format %{ "cb$cmp   $op1, $labl # long" %}
15880   ins_encode %{
15881     Label* L = $labl$$label;
15882     Assembler::Condition cond =
15883       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15884     __ tbr(cond, $op1$$Register, 63, *L);
15885   %}
15886   ins_pipe(pipe_cmp_branch);
15887   ins_short_branch(1);
15888 %}
15889 
15890 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15891   match(If cmp (CmpI op1 op2));
15892   effect(USE labl);
15893 
15894   ins_cost(BRANCH_COST);
15895   format %{ "cb$cmp   $op1, $labl # int" %}
15896   ins_encode %{
15897     Label* L = $labl$$label;
15898     Assembler::Condition cond =
15899       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15900     __ tbr(cond, $op1$$Register, 31, *L);
15901   %}
15902   ins_pipe(pipe_cmp_branch);
15903   ins_short_branch(1);
15904 %}
15905 
15906 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15907   match(If cmp (CmpL (AndL op1 op2) op3));
15908   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15909   effect(USE labl);
15910 
15911   ins_cost(BRANCH_COST);
15912   format %{ "tb$cmp   $op1, $op2, $labl" %}
15913   ins_encode %{
15914     Label* L = $labl$$label;
15915     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15916     int bit = exact_log2_long($op2$$constant);
15917     __ tbr(cond, $op1$$Register, bit, *L);
15918   %}
15919   ins_pipe(pipe_cmp_branch);
15920   ins_short_branch(1);
15921 %}
15922 
15923 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15924   match(If cmp (CmpI (AndI op1 op2) op3));
15925   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15926   effect(USE labl);
15927 
15928   ins_cost(BRANCH_COST);
15929   format %{ "tb$cmp   $op1, $op2, $labl" %}
15930   ins_encode %{
15931     Label* L = $labl$$label;
15932     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15933     int bit = exact_log2((juint)$op2$$constant);
15934     __ tbr(cond, $op1$$Register, bit, *L);
15935   %}
15936   ins_pipe(pipe_cmp_branch);
15937   ins_short_branch(1);
15938 %}
15939 
15940 // And far variants
15941 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15942   match(If cmp (CmpL op1 op2));
15943   effect(USE labl);
15944 
15945   ins_cost(BRANCH_COST);
15946   format %{ "cb$cmp   $op1, $labl # long" %}
15947   ins_encode %{
15948     Label* L = $labl$$label;
15949     Assembler::Condition cond =
15950       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15951     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15952   %}
15953   ins_pipe(pipe_cmp_branch);
15954 %}
15955 
15956 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15957   match(If cmp (CmpI op1 op2));
15958   effect(USE labl);
15959 
15960   ins_cost(BRANCH_COST);
15961   format %{ "cb$cmp   $op1, $labl # int" %}
15962   ins_encode %{
15963     Label* L = $labl$$label;
15964     Assembler::Condition cond =
15965       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15966     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15967   %}
15968   ins_pipe(pipe_cmp_branch);
15969 %}
15970 
15971 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15972   match(If cmp (CmpL (AndL op1 op2) op3));
15973   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15974   effect(USE labl);
15975 
15976   ins_cost(BRANCH_COST);
15977   format %{ "tb$cmp   $op1, $op2, $labl" %}
15978   ins_encode %{
15979     Label* L = $labl$$label;
15980     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15981     int bit = exact_log2_long($op2$$constant);
15982     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15983   %}
15984   ins_pipe(pipe_cmp_branch);
15985 %}
15986 
15987 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15988   match(If cmp (CmpI (AndI op1 op2) op3));
15989   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15990   effect(USE labl);
15991 
15992   ins_cost(BRANCH_COST);
15993   format %{ "tb$cmp   $op1, $op2, $labl" %}
15994   ins_encode %{
15995     Label* L = $labl$$label;
15996     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15997     int bit = exact_log2((juint)$op2$$constant);
15998     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15999   %}
16000   ins_pipe(pipe_cmp_branch);
16001 %}
16002 
16003 // Test bits
16004 
16005 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16006   match(Set cr (CmpL (AndL op1 op2) op3));
16007   predicate(Assembler::operand_valid_for_logical_immediate
16008             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16009 
16010   ins_cost(INSN_COST);
16011   format %{ "tst $op1, $op2 # long" %}
16012   ins_encode %{
16013     __ tst($op1$$Register, $op2$$constant);
16014   %}
16015   ins_pipe(ialu_reg_reg);
16016 %}
16017 
16018 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16019   match(Set cr (CmpI (AndI op1 op2) op3));
16020   predicate(Assembler::operand_valid_for_logical_immediate
16021             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16022 
16023   ins_cost(INSN_COST);
16024   format %{ "tst $op1, $op2 # int" %}
16025   ins_encode %{
16026     __ tstw($op1$$Register, $op2$$constant);
16027   %}
16028   ins_pipe(ialu_reg_reg);
16029 %}
16030 
16031 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16032   match(Set cr (CmpL (AndL op1 op2) op3));
16033 
16034   ins_cost(INSN_COST);
16035   format %{ "tst $op1, $op2 # long" %}
16036   ins_encode %{
16037     __ tst($op1$$Register, $op2$$Register);
16038   %}
16039   ins_pipe(ialu_reg_reg);
16040 %}
16041 
16042 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16043   match(Set cr (CmpI (AndI op1 op2) op3));
16044 
16045   ins_cost(INSN_COST);
16046   format %{ "tstw $op1, $op2 # int" %}
16047   ins_encode %{
16048     __ tstw($op1$$Register, $op2$$Register);
16049   %}
16050   ins_pipe(ialu_reg_reg);
16051 %}
16052 
16053 
16054 // Conditional Far Branch
16055 // Conditional Far Branch Unsigned
16056 // TODO: fixme
16057 
16058 // counted loop end branch near
16059 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16060 %{
16061   match(CountedLoopEnd cmp cr);
16062 
16063   effect(USE lbl);
16064 
16065   ins_cost(BRANCH_COST);
16066   // short variant.
16067   // ins_short_branch(1);
16068   format %{ "b$cmp $lbl \t// counted loop end" %}
16069 
16070   ins_encode(aarch64_enc_br_con(cmp, lbl));
16071 
16072   ins_pipe(pipe_branch);
16073 %}
16074 
16075 // counted loop end branch far
16076 // TODO: fixme
16077 
16078 // ============================================================================
16079 // inlined locking and unlocking
16080 
16081 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16082 %{
16083   predicate(LockingMode != LM_LIGHTWEIGHT);
16084   match(Set cr (FastLock object box));
16085   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16086 
16087   ins_cost(5 * INSN_COST);
16088   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16089 
16090   ins_encode %{
16091     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16092   %}
16093 
16094   ins_pipe(pipe_serial);
16095 %}
16096 
16097 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16098 %{
16099   predicate(LockingMode != LM_LIGHTWEIGHT);
16100   match(Set cr (FastUnlock object box));
16101   effect(TEMP tmp, TEMP tmp2);
16102 
16103   ins_cost(5 * INSN_COST);
16104   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16105 
16106   ins_encode %{
16107     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16108   %}
16109 
16110   ins_pipe(pipe_serial);
16111 %}
16112 
16113 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16114 %{
16115   predicate(LockingMode == LM_LIGHTWEIGHT);
16116   match(Set cr (FastLock object box));
16117   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16118 
16119   ins_cost(5 * INSN_COST);
16120   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16121 
16122   ins_encode %{
16123     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16124   %}
16125 
16126   ins_pipe(pipe_serial);
16127 %}
16128 
16129 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16130 %{
16131   predicate(LockingMode == LM_LIGHTWEIGHT);
16132   match(Set cr (FastUnlock object box));
16133   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16134 
16135   ins_cost(5 * INSN_COST);
16136   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16137 
16138   ins_encode %{
16139     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16140   %}
16141 
16142   ins_pipe(pipe_serial);
16143 %}
16144 
16145 // ============================================================================
16146 // Safepoint Instructions
16147 
16148 // TODO
16149 // provide a near and far version of this code
16150 
16151 instruct safePoint(rFlagsReg cr, iRegP poll)
16152 %{
16153   match(SafePoint poll);
16154   effect(KILL cr);
16155 
16156   format %{
16157     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16158   %}
16159   ins_encode %{
16160     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16161   %}
16162   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16163 %}
16164 
16165 
16166 // ============================================================================
16167 // Procedure Call/Return Instructions
16168 
16169 // Call Java Static Instruction
16170 
16171 instruct CallStaticJavaDirect(method meth)
16172 %{
16173   match(CallStaticJava);
16174 
16175   effect(USE meth);
16176 
16177   ins_cost(CALL_COST);
16178 
16179   format %{ "call,static $meth \t// ==> " %}
16180 
16181   ins_encode(aarch64_enc_java_static_call(meth),
16182              aarch64_enc_call_epilog);
16183 
16184   ins_pipe(pipe_class_call);
16185 %}
16186 
16187 // TO HERE
16188 
16189 // Call Java Dynamic Instruction
16190 instruct CallDynamicJavaDirect(method meth)
16191 %{
16192   match(CallDynamicJava);
16193 
16194   effect(USE meth);
16195 
16196   ins_cost(CALL_COST);
16197 
16198   format %{ "CALL,dynamic $meth \t// ==> " %}
16199 
16200   ins_encode(aarch64_enc_java_dynamic_call(meth),
16201              aarch64_enc_call_epilog);
16202 
16203   ins_pipe(pipe_class_call);
16204 %}
16205 
16206 // Call Runtime Instruction
16207 
16208 instruct CallRuntimeDirect(method meth)
16209 %{
16210   match(CallRuntime);
16211 
16212   effect(USE meth);
16213 
16214   ins_cost(CALL_COST);
16215 
16216   format %{ "CALL, runtime $meth" %}
16217 
16218   ins_encode( aarch64_enc_java_to_runtime(meth) );
16219 
16220   ins_pipe(pipe_class_call);
16221 %}
16222 
16223 // Call Runtime Instruction
16224 
16225 instruct CallLeafDirect(method meth)
16226 %{
16227   match(CallLeaf);
16228 
16229   effect(USE meth);
16230 
16231   ins_cost(CALL_COST);
16232 
16233   format %{ "CALL, runtime leaf $meth" %}
16234 
16235   ins_encode( aarch64_enc_java_to_runtime(meth) );
16236 
16237   ins_pipe(pipe_class_call);
16238 %}
16239 
16240 // Call Runtime Instruction without safepoint and with vector arguments
16241 instruct CallLeafDirectVector(method meth)
16242 %{
16243   match(CallLeafVector);
16244 
16245   effect(USE meth);
16246 
16247   ins_cost(CALL_COST);
16248 
16249   format %{ "CALL, runtime leaf vector $meth" %}
16250 
16251   ins_encode(aarch64_enc_java_to_runtime(meth));
16252 
16253   ins_pipe(pipe_class_call);
16254 %}
16255 
16256 // Call Runtime Instruction
16257 
16258 // entry point is null, target holds the address to call
16259 instruct CallLeafNoFPIndirect(iRegP target)
16260 %{
16261   predicate(n->as_Call()->entry_point() == nullptr);
16262 
16263   match(CallLeafNoFP target);
16264 
16265   ins_cost(CALL_COST);
16266 
16267   format %{ "CALL, runtime leaf nofp indirect $target" %}
16268 
16269   ins_encode %{
16270     __ blr($target$$Register);
16271   %}
16272 
16273   ins_pipe(pipe_class_call);
16274 %}
16275 
16276 instruct CallLeafNoFPDirect(method meth)
16277 %{
16278   predicate(n->as_Call()->entry_point() != nullptr);
16279 
16280   match(CallLeafNoFP);
16281 
16282   effect(USE meth);
16283 
16284   ins_cost(CALL_COST);
16285 
16286   format %{ "CALL, runtime leaf nofp $meth" %}
16287 
16288   ins_encode( aarch64_enc_java_to_runtime(meth) );
16289 
16290   ins_pipe(pipe_class_call);
16291 %}
16292 
16293 // Tail Call; Jump from runtime stub to Java code.
16294 // Also known as an 'interprocedural jump'.
16295 // Target of jump will eventually return to caller.
16296 // TailJump below removes the return address.
16297 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16298 // emitted just above the TailCall which has reset rfp to the caller state.
16299 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16300 %{
16301   match(TailCall jump_target method_ptr);
16302 
16303   ins_cost(CALL_COST);
16304 
16305   format %{ "br $jump_target\t# $method_ptr holds method" %}
16306 
16307   ins_encode(aarch64_enc_tail_call(jump_target));
16308 
16309   ins_pipe(pipe_class_call);
16310 %}
16311 
16312 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16313 %{
16314   match(TailJump jump_target ex_oop);
16315 
16316   ins_cost(CALL_COST);
16317 
16318   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16319 
16320   ins_encode(aarch64_enc_tail_jmp(jump_target));
16321 
16322   ins_pipe(pipe_class_call);
16323 %}
16324 
16325 // Forward exception.
16326 instruct ForwardExceptionjmp()
16327 %{
16328   match(ForwardException);
16329   ins_cost(CALL_COST);
16330 
16331   format %{ "b forward_exception_stub" %}
16332   ins_encode %{
16333     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16334   %}
16335   ins_pipe(pipe_class_call);
16336 %}
16337 
16338 // Create exception oop: created by stack-crawling runtime code.
16339 // Created exception is now available to this handler, and is setup
16340 // just prior to jumping to this handler. No code emitted.
16341 // TODO check
16342 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16343 instruct CreateException(iRegP_R0 ex_oop)
16344 %{
16345   match(Set ex_oop (CreateEx));
16346 
16347   format %{ " -- \t// exception oop; no code emitted" %}
16348 
16349   size(0);
16350 
16351   ins_encode( /*empty*/ );
16352 
16353   ins_pipe(pipe_class_empty);
16354 %}
16355 
16356 // Rethrow exception: The exception oop will come in the first
16357 // argument position. Then JUMP (not call) to the rethrow stub code.
16358 instruct RethrowException() %{
16359   match(Rethrow);
16360   ins_cost(CALL_COST);
16361 
16362   format %{ "b rethrow_stub" %}
16363 
16364   ins_encode( aarch64_enc_rethrow() );
16365 
16366   ins_pipe(pipe_class_call);
16367 %}
16368 
16369 
16370 // Return Instruction
16371 // epilog node loads ret address into lr as part of frame pop
16372 instruct Ret()
16373 %{
16374   match(Return);
16375 
16376   format %{ "ret\t// return register" %}
16377 
16378   ins_encode( aarch64_enc_ret() );
16379 
16380   ins_pipe(pipe_branch);
16381 %}
16382 
16383 // Die now.
16384 instruct ShouldNotReachHere() %{
16385   match(Halt);
16386 
16387   ins_cost(CALL_COST);
16388   format %{ "ShouldNotReachHere" %}
16389 
16390   ins_encode %{
16391     if (is_reachable()) {
16392       __ stop(_halt_reason);
16393     }
16394   %}
16395 
16396   ins_pipe(pipe_class_default);
16397 %}
16398 
16399 // ============================================================================
16400 // Partial Subtype Check
16401 //
16402 // superklass array for an instance of the superklass.  Set a hidden
16403 // internal cache on a hit (cache is checked with exposed code in
16404 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16405 // encoding ALSO sets flags.
16406 
16407 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16408 %{
16409   match(Set result (PartialSubtypeCheck sub super));
16410   predicate(!UseSecondarySupersTable);
16411   effect(KILL cr, KILL temp);
16412 
16413   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16414   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16415 
16416   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16417 
16418   opcode(0x1); // Force zero of result reg on hit
16419 
16420   ins_pipe(pipe_class_memory);
16421 %}
16422 
16423 // Two versions of partialSubtypeCheck, both used when we need to
16424 // search for a super class in the secondary supers array. The first
16425 // is used when we don't know _a priori_ the class being searched
16426 // for. The second, far more common, is used when we do know: this is
16427 // used for instanceof, checkcast, and any case where C2 can determine
16428 // it by constant propagation.
16429 
16430 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16431                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16432                                      rFlagsReg cr)
16433 %{
16434   match(Set result (PartialSubtypeCheck sub super));
16435   predicate(UseSecondarySupersTable);
16436   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16437 
16438   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16439   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16440 
16441   ins_encode %{
16442     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16443                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16444                                          $vtemp$$FloatRegister,
16445                                          $result$$Register, /*L_success*/nullptr);
16446   %}
16447 
16448   ins_pipe(pipe_class_memory);
16449 %}
16450 
16451 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16452                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16453                                        rFlagsReg cr)
16454 %{
16455   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16456   predicate(UseSecondarySupersTable);
16457   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16458 
16459   ins_cost(5 * INSN_COST);  // smaller than the next version
16460   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16461 
16462   ins_encode %{
16463     bool success = false;
16464     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16465     if (InlineSecondarySupersTest) {
16466       success =
16467         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16468                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16469                                                $vtemp$$FloatRegister,
16470                                                $result$$Register,
16471                                                super_klass_slot);
16472     } else {
16473       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16474       success = (call != nullptr);
16475     }
16476     if (!success) {
16477       ciEnv::current()->record_failure("CodeCache is full");
16478       return;
16479     }
16480   %}
16481 
16482   ins_pipe(pipe_class_memory);
16483 %}
16484 
16485 // Intrisics for String.compareTo()
16486 
16487 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16488                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16489 %{
16490   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16491   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16492   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16493 
16494   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16495   ins_encode %{
16496     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16497     __ string_compare($str1$$Register, $str2$$Register,
16498                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16499                       $tmp1$$Register, $tmp2$$Register,
16500                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16501   %}
16502   ins_pipe(pipe_class_memory);
16503 %}
16504 
16505 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16506                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16507 %{
16508   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16509   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16510   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16511 
16512   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16513   ins_encode %{
16514     __ string_compare($str1$$Register, $str2$$Register,
16515                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16516                       $tmp1$$Register, $tmp2$$Register,
16517                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16518   %}
16519   ins_pipe(pipe_class_memory);
16520 %}
16521 
16522 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16523                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16524                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16525 %{
16526   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16527   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16528   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16529          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16530 
16531   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16532   ins_encode %{
16533     __ string_compare($str1$$Register, $str2$$Register,
16534                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16535                       $tmp1$$Register, $tmp2$$Register,
16536                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16537                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16538   %}
16539   ins_pipe(pipe_class_memory);
16540 %}
16541 
16542 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16543                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16544                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16545 %{
16546   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16547   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16548   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16549          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16550 
16551   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16552   ins_encode %{
16553     __ string_compare($str1$$Register, $str2$$Register,
16554                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16555                       $tmp1$$Register, $tmp2$$Register,
16556                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16557                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16558   %}
16559   ins_pipe(pipe_class_memory);
16560 %}
16561 
16562 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16563 // these string_compare variants as NEON register type for convenience so that the prototype of
16564 // string_compare can be shared with all variants.
16565 
16566 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16567                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16568                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16569                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16570 %{
16571   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16572   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16573   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16574          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16575 
16576   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16577   ins_encode %{
16578     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16579     __ string_compare($str1$$Register, $str2$$Register,
16580                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16581                       $tmp1$$Register, $tmp2$$Register,
16582                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16583                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16584                       StrIntrinsicNode::LL);
16585   %}
16586   ins_pipe(pipe_class_memory);
16587 %}
16588 
16589 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16590                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16591                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16592                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16593 %{
16594   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16595   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16596   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16597          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16598 
16599   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16600   ins_encode %{
16601     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16602     __ string_compare($str1$$Register, $str2$$Register,
16603                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16604                       $tmp1$$Register, $tmp2$$Register,
16605                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16606                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16607                       StrIntrinsicNode::LU);
16608   %}
16609   ins_pipe(pipe_class_memory);
16610 %}
16611 
16612 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16613                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16614                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16615                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16616 %{
16617   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16618   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16619   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16620          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16621 
16622   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16623   ins_encode %{
16624     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16625     __ string_compare($str1$$Register, $str2$$Register,
16626                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16627                       $tmp1$$Register, $tmp2$$Register,
16628                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16629                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16630                       StrIntrinsicNode::UL);
16631   %}
16632   ins_pipe(pipe_class_memory);
16633 %}
16634 
16635 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16636                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16637                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16638                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16639 %{
16640   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16641   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16642   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16643          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16644 
16645   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16646   ins_encode %{
16647     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16648     __ string_compare($str1$$Register, $str2$$Register,
16649                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16650                       $tmp1$$Register, $tmp2$$Register,
16651                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16652                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16653                       StrIntrinsicNode::UU);
16654   %}
16655   ins_pipe(pipe_class_memory);
16656 %}
16657 
16658 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16659                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16660                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16661                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16662 %{
16663   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16664   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16665   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16666          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16667          TEMP vtmp0, TEMP vtmp1, KILL cr);
16668   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16669             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16670 
16671   ins_encode %{
16672     __ string_indexof($str1$$Register, $str2$$Register,
16673                       $cnt1$$Register, $cnt2$$Register,
16674                       $tmp1$$Register, $tmp2$$Register,
16675                       $tmp3$$Register, $tmp4$$Register,
16676                       $tmp5$$Register, $tmp6$$Register,
16677                       -1, $result$$Register, StrIntrinsicNode::UU);
16678   %}
16679   ins_pipe(pipe_class_memory);
16680 %}
16681 
16682 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16683                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16684                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16685                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16686 %{
16687   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16688   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16689   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16690          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16691          TEMP vtmp0, TEMP vtmp1, KILL cr);
16692   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16693             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16694 
16695   ins_encode %{
16696     __ string_indexof($str1$$Register, $str2$$Register,
16697                       $cnt1$$Register, $cnt2$$Register,
16698                       $tmp1$$Register, $tmp2$$Register,
16699                       $tmp3$$Register, $tmp4$$Register,
16700                       $tmp5$$Register, $tmp6$$Register,
16701                       -1, $result$$Register, StrIntrinsicNode::LL);
16702   %}
16703   ins_pipe(pipe_class_memory);
16704 %}
16705 
16706 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16707                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16708                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16709                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16710 %{
16711   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16712   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16713   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16714          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16715          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16716   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16717             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16718 
16719   ins_encode %{
16720     __ string_indexof($str1$$Register, $str2$$Register,
16721                       $cnt1$$Register, $cnt2$$Register,
16722                       $tmp1$$Register, $tmp2$$Register,
16723                       $tmp3$$Register, $tmp4$$Register,
16724                       $tmp5$$Register, $tmp6$$Register,
16725                       -1, $result$$Register, StrIntrinsicNode::UL);
16726   %}
16727   ins_pipe(pipe_class_memory);
16728 %}
16729 
16730 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16731                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16732                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16733 %{
16734   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16735   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16736   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16737          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16738   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16739             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16740 
16741   ins_encode %{
16742     int icnt2 = (int)$int_cnt2$$constant;
16743     __ string_indexof($str1$$Register, $str2$$Register,
16744                       $cnt1$$Register, zr,
16745                       $tmp1$$Register, $tmp2$$Register,
16746                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16747                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16748   %}
16749   ins_pipe(pipe_class_memory);
16750 %}
16751 
16752 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16753                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16754                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16755 %{
16756   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16757   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16758   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16759          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16760   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16761             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16762 
16763   ins_encode %{
16764     int icnt2 = (int)$int_cnt2$$constant;
16765     __ string_indexof($str1$$Register, $str2$$Register,
16766                       $cnt1$$Register, zr,
16767                       $tmp1$$Register, $tmp2$$Register,
16768                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16769                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16770   %}
16771   ins_pipe(pipe_class_memory);
16772 %}
16773 
16774 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16775                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16776                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16777 %{
16778   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16779   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16780   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16781          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16782   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16783             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16784 
16785   ins_encode %{
16786     int icnt2 = (int)$int_cnt2$$constant;
16787     __ string_indexof($str1$$Register, $str2$$Register,
16788                       $cnt1$$Register, zr,
16789                       $tmp1$$Register, $tmp2$$Register,
16790                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16791                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16792   %}
16793   ins_pipe(pipe_class_memory);
16794 %}
16795 
16796 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16797                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16798                              iRegINoSp tmp3, rFlagsReg cr)
16799 %{
16800   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16801   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16802   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16803          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16804 
16805   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16806 
16807   ins_encode %{
16808     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16809                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16810                            $tmp3$$Register);
16811   %}
16812   ins_pipe(pipe_class_memory);
16813 %}
16814 
16815 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16816                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16817                               iRegINoSp tmp3, rFlagsReg cr)
16818 %{
16819   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16820   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16821   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16822          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16823 
16824   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16825 
16826   ins_encode %{
16827     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16828                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16829                             $tmp3$$Register);
16830   %}
16831   ins_pipe(pipe_class_memory);
16832 %}
16833 
16834 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16835                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16836                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16837   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16838   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16839   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16840   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16841   ins_encode %{
16842     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16843                                $result$$Register, $ztmp1$$FloatRegister,
16844                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16845                                $ptmp$$PRegister, true /* isL */);
16846   %}
16847   ins_pipe(pipe_class_memory);
16848 %}
16849 
16850 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16851                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16852                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16853   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16854   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16855   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16856   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16857   ins_encode %{
16858     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16859                                $result$$Register, $ztmp1$$FloatRegister,
16860                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16861                                $ptmp$$PRegister, false /* isL */);
16862   %}
16863   ins_pipe(pipe_class_memory);
16864 %}
16865 
16866 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16867                         iRegI_R0 result, rFlagsReg cr)
16868 %{
16869   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16870   match(Set result (StrEquals (Binary str1 str2) cnt));
16871   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16872 
16873   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16874   ins_encode %{
16875     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16876     __ string_equals($str1$$Register, $str2$$Register,
16877                      $result$$Register, $cnt$$Register);
16878   %}
16879   ins_pipe(pipe_class_memory);
16880 %}
16881 
16882 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16883                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16884                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16885                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16886                        iRegP_R10 tmp, rFlagsReg cr)
16887 %{
16888   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16889   match(Set result (AryEq ary1 ary2));
16890   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16891          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16892          TEMP vtmp6, TEMP vtmp7, KILL cr);
16893 
16894   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16895   ins_encode %{
16896     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16897                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16898                                    $result$$Register, $tmp$$Register, 1);
16899     if (tpc == nullptr) {
16900       ciEnv::current()->record_failure("CodeCache is full");
16901       return;
16902     }
16903   %}
16904   ins_pipe(pipe_class_memory);
16905 %}
16906 
16907 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16908                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16909                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16910                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16911                        iRegP_R10 tmp, rFlagsReg cr)
16912 %{
16913   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16914   match(Set result (AryEq ary1 ary2));
16915   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16916          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16917          TEMP vtmp6, TEMP vtmp7, KILL cr);
16918 
16919   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16920   ins_encode %{
16921     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16922                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16923                                    $result$$Register, $tmp$$Register, 2);
16924     if (tpc == nullptr) {
16925       ciEnv::current()->record_failure("CodeCache is full");
16926       return;
16927     }
16928   %}
16929   ins_pipe(pipe_class_memory);
16930 %}
16931 
16932 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16933                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16934                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16935                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16936 %{
16937   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16938   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16939          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16940 
16941   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
16942   ins_encode %{
16943     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16944                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16945                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16946                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16947                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16948                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16949                                      (BasicType)$basic_type$$constant);
16950     if (tpc == nullptr) {
16951       ciEnv::current()->record_failure("CodeCache is full");
16952       return;
16953     }
16954   %}
16955   ins_pipe(pipe_class_memory);
16956 %}
16957 
16958 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16959 %{
16960   match(Set result (CountPositives ary1 len));
16961   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16962   format %{ "count positives byte[] $ary1,$len -> $result" %}
16963   ins_encode %{
16964     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16965     if (tpc == nullptr) {
16966       ciEnv::current()->record_failure("CodeCache is full");
16967       return;
16968     }
16969   %}
16970   ins_pipe( pipe_slow );
16971 %}
16972 
16973 // fast char[] to byte[] compression
16974 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16975                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16976                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16977                          iRegI_R0 result, rFlagsReg cr)
16978 %{
16979   match(Set result (StrCompressedCopy src (Binary dst len)));
16980   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16981          USE_KILL src, USE_KILL dst, USE len, KILL cr);
16982 
16983   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16984   ins_encode %{
16985     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16986                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16987                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16988                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16989   %}
16990   ins_pipe(pipe_slow);
16991 %}
16992 
16993 // fast byte[] to char[] inflation
16994 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16995                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16996                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16997 %{
16998   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16999   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17000          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17001          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17002 
17003   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17004   ins_encode %{
17005     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17006                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17007                                         $vtmp2$$FloatRegister, $tmp$$Register);
17008     if (tpc == nullptr) {
17009       ciEnv::current()->record_failure("CodeCache is full");
17010       return;
17011     }
17012   %}
17013   ins_pipe(pipe_class_memory);
17014 %}
17015 
17016 // encode char[] to byte[] in ISO_8859_1
17017 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17018                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17019                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17020                           iRegI_R0 result, rFlagsReg cr)
17021 %{
17022   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17023   match(Set result (EncodeISOArray src (Binary dst len)));
17024   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17025          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17026 
17027   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17028   ins_encode %{
17029     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17030                         $result$$Register, false,
17031                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17032                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17033                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17034   %}
17035   ins_pipe(pipe_class_memory);
17036 %}
17037 
17038 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17039                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17040                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17041                             iRegI_R0 result, rFlagsReg cr)
17042 %{
17043   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17044   match(Set result (EncodeISOArray src (Binary dst len)));
17045   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17046          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17047 
17048   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17049   ins_encode %{
17050     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17051                         $result$$Register, true,
17052                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17053                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17054                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17055   %}
17056   ins_pipe(pipe_class_memory);
17057 %}
17058 
17059 //----------------------------- CompressBits/ExpandBits ------------------------
17060 
17061 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17062                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17063   match(Set dst (CompressBits src mask));
17064   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17065   format %{ "mov    $tsrc, $src\n\t"
17066             "mov    $tmask, $mask\n\t"
17067             "bext   $tdst, $tsrc, $tmask\n\t"
17068             "mov    $dst, $tdst"
17069           %}
17070   ins_encode %{
17071     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17072     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17073     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17074     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17075   %}
17076   ins_pipe(pipe_slow);
17077 %}
17078 
17079 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17080                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17081   match(Set dst (CompressBits (LoadI mem) mask));
17082   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17083   format %{ "ldrs   $tsrc, $mem\n\t"
17084             "ldrs   $tmask, $mask\n\t"
17085             "bext   $tdst, $tsrc, $tmask\n\t"
17086             "mov    $dst, $tdst"
17087           %}
17088   ins_encode %{
17089     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17090               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17091     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17092     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17093     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17094   %}
17095   ins_pipe(pipe_slow);
17096 %}
17097 
17098 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17099                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17100   match(Set dst (CompressBits src mask));
17101   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17102   format %{ "mov    $tsrc, $src\n\t"
17103             "mov    $tmask, $mask\n\t"
17104             "bext   $tdst, $tsrc, $tmask\n\t"
17105             "mov    $dst, $tdst"
17106           %}
17107   ins_encode %{
17108     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17109     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17110     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17111     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17112   %}
17113   ins_pipe(pipe_slow);
17114 %}
17115 
17116 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17117                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17118   match(Set dst (CompressBits (LoadL mem) mask));
17119   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17120   format %{ "ldrd   $tsrc, $mem\n\t"
17121             "ldrd   $tmask, $mask\n\t"
17122             "bext   $tdst, $tsrc, $tmask\n\t"
17123             "mov    $dst, $tdst"
17124           %}
17125   ins_encode %{
17126     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17127               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17128     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17129     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17130     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17131   %}
17132   ins_pipe(pipe_slow);
17133 %}
17134 
17135 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17136                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17137   match(Set dst (ExpandBits src mask));
17138   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17139   format %{ "mov    $tsrc, $src\n\t"
17140             "mov    $tmask, $mask\n\t"
17141             "bdep   $tdst, $tsrc, $tmask\n\t"
17142             "mov    $dst, $tdst"
17143           %}
17144   ins_encode %{
17145     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17146     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17147     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17148     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17149   %}
17150   ins_pipe(pipe_slow);
17151 %}
17152 
17153 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17154                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17155   match(Set dst (ExpandBits (LoadI mem) mask));
17156   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17157   format %{ "ldrs   $tsrc, $mem\n\t"
17158             "ldrs   $tmask, $mask\n\t"
17159             "bdep   $tdst, $tsrc, $tmask\n\t"
17160             "mov    $dst, $tdst"
17161           %}
17162   ins_encode %{
17163     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17164               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17165     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17166     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17167     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17168   %}
17169   ins_pipe(pipe_slow);
17170 %}
17171 
17172 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17173                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17174   match(Set dst (ExpandBits src mask));
17175   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17176   format %{ "mov    $tsrc, $src\n\t"
17177             "mov    $tmask, $mask\n\t"
17178             "bdep   $tdst, $tsrc, $tmask\n\t"
17179             "mov    $dst, $tdst"
17180           %}
17181   ins_encode %{
17182     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17183     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17184     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17185     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17186   %}
17187   ins_pipe(pipe_slow);
17188 %}
17189 
17190 
17191 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17192                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17193   match(Set dst (ExpandBits (LoadL mem) mask));
17194   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17195   format %{ "ldrd   $tsrc, $mem\n\t"
17196             "ldrd   $tmask, $mask\n\t"
17197             "bdep   $tdst, $tsrc, $tmask\n\t"
17198             "mov    $dst, $tdst"
17199           %}
17200   ins_encode %{
17201     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17202               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17203     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17204     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17205     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17206   %}
17207   ins_pipe(pipe_slow);
17208 %}
17209 
17210 // ============================================================================
17211 // This name is KNOWN by the ADLC and cannot be changed.
17212 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17213 // for this guy.
17214 instruct tlsLoadP(thread_RegP dst)
17215 %{
17216   match(Set dst (ThreadLocal));
17217 
17218   ins_cost(0);
17219 
17220   format %{ " -- \t// $dst=Thread::current(), empty" %}
17221 
17222   size(0);
17223 
17224   ins_encode( /*empty*/ );
17225 
17226   ins_pipe(pipe_class_empty);
17227 %}
17228 
17229 //----------PEEPHOLE RULES-----------------------------------------------------
17230 // These must follow all instruction definitions as they use the names
17231 // defined in the instructions definitions.
17232 //
17233 // peepmatch ( root_instr_name [preceding_instruction]* );
17234 //
17235 // peepconstraint %{
17236 // (instruction_number.operand_name relational_op instruction_number.operand_name
17237 //  [, ...] );
17238 // // instruction numbers are zero-based using left to right order in peepmatch
17239 //
17240 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17241 // // provide an instruction_number.operand_name for each operand that appears
17242 // // in the replacement instruction's match rule
17243 //
17244 // ---------VM FLAGS---------------------------------------------------------
17245 //
17246 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17247 //
17248 // Each peephole rule is given an identifying number starting with zero and
17249 // increasing by one in the order seen by the parser.  An individual peephole
17250 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17251 // on the command-line.
17252 //
17253 // ---------CURRENT LIMITATIONS----------------------------------------------
17254 //
17255 // Only match adjacent instructions in same basic block
17256 // Only equality constraints
17257 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17258 // Only one replacement instruction
17259 //
17260 // ---------EXAMPLE----------------------------------------------------------
17261 //
17262 // // pertinent parts of existing instructions in architecture description
17263 // instruct movI(iRegINoSp dst, iRegI src)
17264 // %{
17265 //   match(Set dst (CopyI src));
17266 // %}
17267 //
17268 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17269 // %{
17270 //   match(Set dst (AddI dst src));
17271 //   effect(KILL cr);
17272 // %}
17273 //
17274 // // Change (inc mov) to lea
17275 // peephole %{
17276 //   // increment preceded by register-register move
17277 //   peepmatch ( incI_iReg movI );
17278 //   // require that the destination register of the increment
17279 //   // match the destination register of the move
17280 //   peepconstraint ( 0.dst == 1.dst );
17281 //   // construct a replacement instruction that sets
17282 //   // the destination to ( move's source register + one )
17283 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17284 // %}
17285 //
17286 
17287 // Implementation no longer uses movX instructions since
17288 // machine-independent system no longer uses CopyX nodes.
17289 //
17290 // peephole
17291 // %{
17292 //   peepmatch (incI_iReg movI);
17293 //   peepconstraint (0.dst == 1.dst);
17294 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17295 // %}
17296 
17297 // peephole
17298 // %{
17299 //   peepmatch (decI_iReg movI);
17300 //   peepconstraint (0.dst == 1.dst);
17301 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17302 // %}
17303 
17304 // peephole
17305 // %{
17306 //   peepmatch (addI_iReg_imm movI);
17307 //   peepconstraint (0.dst == 1.dst);
17308 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17309 // %}
17310 
17311 // peephole
17312 // %{
17313 //   peepmatch (incL_iReg movL);
17314 //   peepconstraint (0.dst == 1.dst);
17315 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17316 // %}
17317 
17318 // peephole
17319 // %{
17320 //   peepmatch (decL_iReg movL);
17321 //   peepconstraint (0.dst == 1.dst);
17322 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17323 // %}
17324 
17325 // peephole
17326 // %{
17327 //   peepmatch (addL_iReg_imm movL);
17328 //   peepconstraint (0.dst == 1.dst);
17329 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17330 // %}
17331 
17332 // peephole
17333 // %{
17334 //   peepmatch (addP_iReg_imm movP);
17335 //   peepconstraint (0.dst == 1.dst);
17336 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17337 // %}
17338 
17339 // // Change load of spilled value to only a spill
17340 // instruct storeI(memory mem, iRegI src)
17341 // %{
17342 //   match(Set mem (StoreI mem src));
17343 // %}
17344 //
17345 // instruct loadI(iRegINoSp dst, memory mem)
17346 // %{
17347 //   match(Set dst (LoadI mem));
17348 // %}
17349 //
17350 
17351 //----------SMARTSPILL RULES---------------------------------------------------
17352 // These must follow all instruction definitions as they use the names
17353 // defined in the instructions definitions.
17354 
17355 // Local Variables:
17356 // mode: c++
17357 // End: