1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1134 
 1135 class CallStubImpl {
 1136 
 1137   //--------------------------------------------------------------
 1138   //---<  Used for optimization in Compile::shorten_branches  >---
 1139   //--------------------------------------------------------------
 1140 
 1141  public:
 1142   // Size of call trampoline stub.
 1143   static uint size_call_trampoline() {
 1144     return 0; // no call trampolines on this platform
 1145   }
 1146 
 1147   // number of relocations needed by a call trampoline stub
 1148   static uint reloc_call_trampoline() {
 1149     return 0; // no call trampolines on this platform
 1150   }
 1151 };
 1152 
 1153 class HandlerImpl {
 1154 
 1155  public:
 1156 
 1157   static int emit_exception_handler(C2_MacroAssembler *masm);
 1158   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1159 
 1160   static uint size_exception_handler() {
 1161     return MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 
 1164   static uint size_deopt_handler() {
 1165     // count one adr and one far branch instruction
 1166     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1167   }
 1168 };
 1169 
 1170 class Node::PD {
 1171 public:
 1172   enum NodeFlags {
 1173     _last_flag = Node::_last_flag
 1174   };
 1175 };
 1176 
 1177   bool is_CAS(int opcode, bool maybe_volatile);
 1178 
 1179   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1180 
 1181   bool unnecessary_acquire(const Node *barrier);
 1182   bool needs_acquiring_load(const Node *load);
 1183 
 1184   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1185 
 1186   bool unnecessary_release(const Node *barrier);
 1187   bool unnecessary_volatile(const Node *barrier);
 1188   bool needs_releasing_store(const Node *store);
 1189 
 1190   // predicate controlling translation of CompareAndSwapX
 1191   bool needs_acquiring_load_exclusive(const Node *load);
 1192 
 1193   // predicate controlling addressing modes
 1194   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1195 
 1196   // Convert BootTest condition to Assembler condition.
 1197   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1198   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1199 %}
 1200 
 1201 source %{
 1202 
 1203   // Derived RegMask with conditionally allocatable registers
 1204 
 1205   void PhaseOutput::pd_perform_mach_node_analysis() {
 1206   }
 1207 
 1208   int MachNode::pd_alignment_required() const {
 1209     return 1;
 1210   }
 1211 
 1212   int MachNode::compute_padding(int current_offset) const {
 1213     return 0;
 1214   }
 1215 
 1216   RegMask _ANY_REG32_mask;
 1217   RegMask _ANY_REG_mask;
 1218   RegMask _PTR_REG_mask;
 1219   RegMask _NO_SPECIAL_REG32_mask;
 1220   RegMask _NO_SPECIAL_REG_mask;
 1221   RegMask _NO_SPECIAL_PTR_REG_mask;
 1222   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1223 
 1224   void reg_mask_init() {
 1225     // We derive below RegMask(s) from the ones which are auto-generated from
 1226     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1227     // registers conditionally reserved.
 1228 
 1229     _ANY_REG32_mask = _ALL_REG32_mask;
 1230     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1231 
 1232     _ANY_REG_mask = _ALL_REG_mask;
 1233 
 1234     _PTR_REG_mask = _ALL_REG_mask;
 1235 
 1236     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1237     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1238 
 1239     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1240     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1241 
 1242     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1243     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1244 
 1245     // r27 is not allocatable when compressed oops is on and heapbase is not
 1246     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1247     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1251     }
 1252 
 1253     // r29 is not allocatable when PreserveFramePointer is on
 1254     if (PreserveFramePointer) {
 1255       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1256       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1257       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1258     }
 1259 
 1260     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1261     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1262   }
 1263 
 1264   // Optimizaton of volatile gets and puts
 1265   // -------------------------------------
 1266   //
 1267   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1268   // use to implement volatile reads and writes. For a volatile read
 1269   // we simply need
 1270   //
 1271   //   ldar<x>
 1272   //
 1273   // and for a volatile write we need
 1274   //
 1275   //   stlr<x>
 1276   //
 1277   // Alternatively, we can implement them by pairing a normal
 1278   // load/store with a memory barrier. For a volatile read we need
 1279   //
 1280   //   ldr<x>
 1281   //   dmb ishld
 1282   //
 1283   // for a volatile write
 1284   //
 1285   //   dmb ish
 1286   //   str<x>
 1287   //   dmb ish
 1288   //
 1289   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1290   // sequences. These are normally translated to an instruction
 1291   // sequence like the following
 1292   //
 1293   //   dmb      ish
 1294   // retry:
 1295   //   ldxr<x>   rval raddr
 1296   //   cmp       rval rold
 1297   //   b.ne done
 1298   //   stlxr<x>  rval, rnew, rold
 1299   //   cbnz      rval retry
 1300   // done:
 1301   //   cset      r0, eq
 1302   //   dmb ishld
 1303   //
 1304   // Note that the exclusive store is already using an stlxr
 1305   // instruction. That is required to ensure visibility to other
 1306   // threads of the exclusive write (assuming it succeeds) before that
 1307   // of any subsequent writes.
 1308   //
 1309   // The following instruction sequence is an improvement on the above
 1310   //
 1311   // retry:
 1312   //   ldaxr<x>  rval raddr
 1313   //   cmp       rval rold
 1314   //   b.ne done
 1315   //   stlxr<x>  rval, rnew, rold
 1316   //   cbnz      rval retry
 1317   // done:
 1318   //   cset      r0, eq
 1319   //
 1320   // We don't need the leading dmb ish since the stlxr guarantees
 1321   // visibility of prior writes in the case that the swap is
 1322   // successful. Crucially we don't have to worry about the case where
 1323   // the swap is not successful since no valid program should be
 1324   // relying on visibility of prior changes by the attempting thread
 1325   // in the case where the CAS fails.
 1326   //
 1327   // Similarly, we don't need the trailing dmb ishld if we substitute
 1328   // an ldaxr instruction since that will provide all the guarantees we
 1329   // require regarding observation of changes made by other threads
 1330   // before any change to the CAS address observed by the load.
 1331   //
 1332   // In order to generate the desired instruction sequence we need to
 1333   // be able to identify specific 'signature' ideal graph node
 1334   // sequences which i) occur as a translation of a volatile reads or
 1335   // writes or CAS operations and ii) do not occur through any other
 1336   // translation or graph transformation. We can then provide
 1337   // alternative aldc matching rules which translate these node
 1338   // sequences to the desired machine code sequences. Selection of the
 1339   // alternative rules can be implemented by predicates which identify
 1340   // the relevant node sequences.
 1341   //
 1342   // The ideal graph generator translates a volatile read to the node
 1343   // sequence
 1344   //
 1345   //   LoadX[mo_acquire]
 1346   //   MemBarAcquire
 1347   //
 1348   // As a special case when using the compressed oops optimization we
 1349   // may also see this variant
 1350   //
 1351   //   LoadN[mo_acquire]
 1352   //   DecodeN
 1353   //   MemBarAcquire
 1354   //
 1355   // A volatile write is translated to the node sequence
 1356   //
 1357   //   MemBarRelease
 1358   //   StoreX[mo_release] {CardMark}-optional
 1359   //   MemBarVolatile
 1360   //
 1361   // n.b. the above node patterns are generated with a strict
 1362   // 'signature' configuration of input and output dependencies (see
 1363   // the predicates below for exact details). The card mark may be as
 1364   // simple as a few extra nodes or, in a few GC configurations, may
 1365   // include more complex control flow between the leading and
 1366   // trailing memory barriers. However, whatever the card mark
 1367   // configuration these signatures are unique to translated volatile
 1368   // reads/stores -- they will not appear as a result of any other
 1369   // bytecode translation or inlining nor as a consequence of
 1370   // optimizing transforms.
 1371   //
 1372   // We also want to catch inlined unsafe volatile gets and puts and
 1373   // be able to implement them using either ldar<x>/stlr<x> or some
 1374   // combination of ldr<x>/stlr<x> and dmb instructions.
 1375   //
 1376   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1377   // normal volatile put node sequence containing an extra cpuorder
 1378   // membar
 1379   //
 1380   //   MemBarRelease
 1381   //   MemBarCPUOrder
 1382   //   StoreX[mo_release] {CardMark}-optional
 1383   //   MemBarCPUOrder
 1384   //   MemBarVolatile
 1385   //
 1386   // n.b. as an aside, a cpuorder membar is not itself subject to
 1387   // matching and translation by adlc rules.  However, the rule
 1388   // predicates need to detect its presence in order to correctly
 1389   // select the desired adlc rules.
 1390   //
 1391   // Inlined unsafe volatile gets manifest as a slightly different
 1392   // node sequence to a normal volatile get because of the
 1393   // introduction of some CPUOrder memory barriers to bracket the
 1394   // Load. However, but the same basic skeleton of a LoadX feeding a
 1395   // MemBarAcquire, possibly through an optional DecodeN, is still
 1396   // present
 1397   //
 1398   //   MemBarCPUOrder
 1399   //        ||       \\
 1400   //   MemBarCPUOrder LoadX[mo_acquire]
 1401   //        ||            |
 1402   //        ||       {DecodeN} optional
 1403   //        ||       /
 1404   //     MemBarAcquire
 1405   //
 1406   // In this case the acquire membar does not directly depend on the
 1407   // load. However, we can be sure that the load is generated from an
 1408   // inlined unsafe volatile get if we see it dependent on this unique
 1409   // sequence of membar nodes. Similarly, given an acquire membar we
 1410   // can know that it was added because of an inlined unsafe volatile
 1411   // get if it is fed and feeds a cpuorder membar and if its feed
 1412   // membar also feeds an acquiring load.
 1413   //
 1414   // Finally an inlined (Unsafe) CAS operation is translated to the
 1415   // following ideal graph
 1416   //
 1417   //   MemBarRelease
 1418   //   MemBarCPUOrder
 1419   //   CompareAndSwapX {CardMark}-optional
 1420   //   MemBarCPUOrder
 1421   //   MemBarAcquire
 1422   //
 1423   // So, where we can identify these volatile read and write
 1424   // signatures we can choose to plant either of the above two code
 1425   // sequences. For a volatile read we can simply plant a normal
 1426   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1427   // also choose to inhibit translation of the MemBarAcquire and
 1428   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1429   //
 1430   // When we recognise a volatile store signature we can choose to
 1431   // plant at a dmb ish as a translation for the MemBarRelease, a
 1432   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1433   // Alternatively, we can inhibit translation of the MemBarRelease
 1434   // and MemBarVolatile and instead plant a simple stlr<x>
 1435   // instruction.
 1436   //
 1437   // when we recognise a CAS signature we can choose to plant a dmb
 1438   // ish as a translation for the MemBarRelease, the conventional
 1439   // macro-instruction sequence for the CompareAndSwap node (which
 1440   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1441   // Alternatively, we can elide generation of the dmb instructions
 1442   // and plant the alternative CompareAndSwap macro-instruction
 1443   // sequence (which uses ldaxr<x>).
 1444   //
 1445   // Of course, the above only applies when we see these signature
 1446   // configurations. We still want to plant dmb instructions in any
 1447   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1448   // MemBarVolatile. For example, at the end of a constructor which
 1449   // writes final/volatile fields we will see a MemBarRelease
 1450   // instruction and this needs a 'dmb ish' lest we risk the
 1451   // constructed object being visible without making the
 1452   // final/volatile field writes visible.
 1453   //
 1454   // n.b. the translation rules below which rely on detection of the
 1455   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1456   // If we see anything other than the signature configurations we
 1457   // always just translate the loads and stores to ldr<x> and str<x>
 1458   // and translate acquire, release and volatile membars to the
 1459   // relevant dmb instructions.
 1460   //
 1461 
 1462   // is_CAS(int opcode, bool maybe_volatile)
 1463   //
 1464   // return true if opcode is one of the possible CompareAndSwapX
 1465   // values otherwise false.
 1466 
 1467   bool is_CAS(int opcode, bool maybe_volatile)
 1468   {
 1469     switch(opcode) {
 1470       // We handle these
 1471     case Op_CompareAndSwapI:
 1472     case Op_CompareAndSwapL:
 1473     case Op_CompareAndSwapP:
 1474     case Op_CompareAndSwapN:
 1475     case Op_ShenandoahCompareAndSwapP:
 1476     case Op_ShenandoahCompareAndSwapN:
 1477     case Op_CompareAndSwapB:
 1478     case Op_CompareAndSwapS:
 1479     case Op_GetAndSetI:
 1480     case Op_GetAndSetL:
 1481     case Op_GetAndSetP:
 1482     case Op_GetAndSetN:
 1483     case Op_GetAndAddI:
 1484     case Op_GetAndAddL:
 1485       return true;
 1486     case Op_CompareAndExchangeI:
 1487     case Op_CompareAndExchangeN:
 1488     case Op_CompareAndExchangeB:
 1489     case Op_CompareAndExchangeS:
 1490     case Op_CompareAndExchangeL:
 1491     case Op_CompareAndExchangeP:
 1492     case Op_WeakCompareAndSwapB:
 1493     case Op_WeakCompareAndSwapS:
 1494     case Op_WeakCompareAndSwapI:
 1495     case Op_WeakCompareAndSwapL:
 1496     case Op_WeakCompareAndSwapP:
 1497     case Op_WeakCompareAndSwapN:
 1498     case Op_ShenandoahWeakCompareAndSwapP:
 1499     case Op_ShenandoahWeakCompareAndSwapN:
 1500     case Op_ShenandoahCompareAndExchangeP:
 1501     case Op_ShenandoahCompareAndExchangeN:
 1502       return maybe_volatile;
 1503     default:
 1504       return false;
 1505     }
 1506   }
 1507 
 1508   // helper to determine the maximum number of Phi nodes we may need to
 1509   // traverse when searching from a card mark membar for the merge mem
 1510   // feeding a trailing membar or vice versa
 1511 
 1512 // predicates controlling emit of ldr<x>/ldar<x>
 1513 
 1514 bool unnecessary_acquire(const Node *barrier)
 1515 {
 1516   assert(barrier->is_MemBar(), "expecting a membar");
 1517 
 1518   MemBarNode* mb = barrier->as_MemBar();
 1519 
 1520   if (mb->trailing_load()) {
 1521     return true;
 1522   }
 1523 
 1524   if (mb->trailing_load_store()) {
 1525     Node* load_store = mb->in(MemBarNode::Precedent);
 1526     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1527     return is_CAS(load_store->Opcode(), true);
 1528   }
 1529 
 1530   return false;
 1531 }
 1532 
 1533 bool needs_acquiring_load(const Node *n)
 1534 {
 1535   assert(n->is_Load(), "expecting a load");
 1536   LoadNode *ld = n->as_Load();
 1537   return ld->is_acquire();
 1538 }
 1539 
 1540 bool unnecessary_release(const Node *n)
 1541 {
 1542   assert((n->is_MemBar() &&
 1543           n->Opcode() == Op_MemBarRelease),
 1544          "expecting a release membar");
 1545 
 1546   MemBarNode *barrier = n->as_MemBar();
 1547   if (!barrier->leading()) {
 1548     return false;
 1549   } else {
 1550     Node* trailing = barrier->trailing_membar();
 1551     MemBarNode* trailing_mb = trailing->as_MemBar();
 1552     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1553     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1554 
 1555     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1556     if (mem->is_Store()) {
 1557       assert(mem->as_Store()->is_release(), "");
 1558       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1559       return true;
 1560     } else {
 1561       assert(mem->is_LoadStore(), "");
 1562       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1563       return is_CAS(mem->Opcode(), true);
 1564     }
 1565   }
 1566   return false;
 1567 }
 1568 
 1569 bool unnecessary_volatile(const Node *n)
 1570 {
 1571   // assert n->is_MemBar();
 1572   MemBarNode *mbvol = n->as_MemBar();
 1573 
 1574   bool release = mbvol->trailing_store();
 1575   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1576 #ifdef ASSERT
 1577   if (release) {
 1578     Node* leading = mbvol->leading_membar();
 1579     assert(leading->Opcode() == Op_MemBarRelease, "");
 1580     assert(leading->as_MemBar()->leading_store(), "");
 1581     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1582   }
 1583 #endif
 1584 
 1585   return release;
 1586 }
 1587 
 1588 // predicates controlling emit of str<x>/stlr<x>
 1589 
 1590 bool needs_releasing_store(const Node *n)
 1591 {
 1592   // assert n->is_Store();
 1593   StoreNode *st = n->as_Store();
 1594   return st->trailing_membar() != nullptr;
 1595 }
 1596 
 1597 // predicate controlling translation of CAS
 1598 //
 1599 // returns true if CAS needs to use an acquiring load otherwise false
 1600 
 1601 bool needs_acquiring_load_exclusive(const Node *n)
 1602 {
 1603   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1604   LoadStoreNode* ldst = n->as_LoadStore();
 1605   if (is_CAS(n->Opcode(), false)) {
 1606     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1607   } else {
 1608     return ldst->trailing_membar() != nullptr;
 1609   }
 1610 
 1611   // so we can just return true here
 1612   return true;
 1613 }
 1614 
 1615 #define __ masm->
 1616 
 1617 // advance declarations for helper functions to convert register
 1618 // indices to register objects
 1619 
 1620 // the ad file has to provide implementations of certain methods
 1621 // expected by the generic code
 1622 //
 1623 // REQUIRED FUNCTIONALITY
 1624 
 1625 //=============================================================================
 1626 
 1627 // !!!!! Special hack to get all types of calls to specify the byte offset
 1628 //       from the start of the call to the point where the return address
 1629 //       will point.
 1630 
 1631 int MachCallStaticJavaNode::ret_addr_offset()
 1632 {
 1633   // call should be a simple bl
 1634   int off = 4;
 1635   return off;
 1636 }
 1637 
 1638 int MachCallDynamicJavaNode::ret_addr_offset()
 1639 {
 1640   return 16; // movz, movk, movk, bl
 1641 }
 1642 
 1643 int MachCallRuntimeNode::ret_addr_offset() {
 1644   // for generated stubs the call will be
 1645   //   bl(addr)
 1646   // or with far branches
 1647   //   bl(trampoline_stub)
 1648   // for real runtime callouts it will be six instructions
 1649   // see aarch64_enc_java_to_runtime
 1650   //   adr(rscratch2, retaddr)
 1651   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1652   //   lea(rscratch1, RuntimeAddress(addr)
 1653   //   blr(rscratch1)
 1654   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1655   if (cb) {
 1656     return 1 * NativeInstruction::instruction_size;
 1657   } else {
 1658     return 6 * NativeInstruction::instruction_size;
 1659   }
 1660 }
 1661 
 1662 //=============================================================================
 1663 
 1664 #ifndef PRODUCT
 1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1666   st->print("BREAKPOINT");
 1667 }
 1668 #endif
 1669 
 1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1671   __ brk(0);
 1672 }
 1673 
 1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1675   return MachNode::size(ra_);
 1676 }
 1677 
 1678 //=============================================================================
 1679 
 1680 #ifndef PRODUCT
 1681   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1682     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1683   }
 1684 #endif
 1685 
 1686   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1687     for (int i = 0; i < _count; i++) {
 1688       __ nop();
 1689     }
 1690   }
 1691 
 1692   uint MachNopNode::size(PhaseRegAlloc*) const {
 1693     return _count * NativeInstruction::instruction_size;
 1694   }
 1695 
 1696 //=============================================================================
 1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1698 
 1699 int ConstantTable::calculate_table_base_offset() const {
 1700   return 0;  // absolute addressing, no offset
 1701 }
 1702 
 1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1705   ShouldNotReachHere();
 1706 }
 1707 
 1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1709   // Empty encoding
 1710 }
 1711 
 1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1713   return 0;
 1714 }
 1715 
 1716 #ifndef PRODUCT
 1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1718   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1719 }
 1720 #endif
 1721 
 1722 #ifndef PRODUCT
 1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1724   Compile* C = ra_->C;
 1725 
 1726   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1727 
 1728   if (C->output()->need_stack_bang(framesize))
 1729     st->print("# stack bang size=%d\n\t", framesize);
 1730 
 1731   if (VM_Version::use_rop_protection()) {
 1732     st->print("ldr  zr, [lr]\n\t");
 1733     st->print("paciaz\n\t");
 1734   }
 1735   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1736     st->print("sub  sp, sp, #%d\n\t", framesize);
 1737     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1738     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1739   } else {
 1740     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1741     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1742     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1743     st->print("sub  sp, sp, rscratch1");
 1744   }
 1745   if (C->stub_function() == nullptr) {
 1746     st->print("\n\t");
 1747     st->print("ldr  rscratch1, [guard]\n\t");
 1748     st->print("dmb ishld\n\t");
 1749     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1750     st->print("cmp  rscratch1, rscratch2\n\t");
 1751     st->print("b.eq skip");
 1752     st->print("\n\t");
 1753     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1754     st->print("b skip\n\t");
 1755     st->print("guard: int\n\t");
 1756     st->print("\n\t");
 1757     st->print("skip:\n\t");
 1758   }
 1759 }
 1760 #endif
 1761 
 1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1763   Compile* C = ra_->C;
 1764 
 1765   // n.b. frame size includes space for return pc and rfp
 1766   const int framesize = C->output()->frame_size_in_bytes();
 1767 
 1768   // insert a nop at the start of the prolog so we can patch in a
 1769   // branch if we need to invalidate the method later
 1770   __ nop();
 1771 
 1772   if (C->clinit_barrier_on_entry()) {
 1773     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1774 
 1775     Label L_skip_barrier;
 1776 
 1777     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1778     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1779     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1780     __ bind(L_skip_barrier);
 1781   }
 1782 
 1783   if (C->max_vector_size() > 0) {
 1784     __ reinitialize_ptrue();
 1785   }
 1786 
 1787   int bangsize = C->output()->bang_size_in_bytes();
 1788   if (C->output()->need_stack_bang(bangsize))
 1789     __ generate_stack_overflow_check(bangsize);
 1790 
 1791   __ build_frame(framesize);
 1792 
 1793   if (C->stub_function() == nullptr) {
 1794     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1795     // Dummy labels for just measuring the code size
 1796     Label dummy_slow_path;
 1797     Label dummy_continuation;
 1798     Label dummy_guard;
 1799     Label* slow_path = &dummy_slow_path;
 1800     Label* continuation = &dummy_continuation;
 1801     Label* guard = &dummy_guard;
 1802     if (!Compile::current()->output()->in_scratch_emit_size()) {
 1803       // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1804       C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1805       Compile::current()->output()->add_stub(stub);
 1806       slow_path = &stub->entry();
 1807       continuation = &stub->continuation();
 1808       guard = &stub->guard();
 1809     }
 1810     // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1811     bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
 1812   }
 1813 
 1814   if (VerifyStackAtCalls) {
 1815     Unimplemented();
 1816   }
 1817 
 1818   C->output()->set_frame_complete(__ offset());
 1819 
 1820   if (C->has_mach_constant_base_node()) {
 1821     // NOTE: We set the table base offset here because users might be
 1822     // emitted before MachConstantBaseNode.
 1823     ConstantTable& constant_table = C->output()->constant_table();
 1824     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1825   }
 1826 }
 1827 
 1828 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1829 {
 1830   return MachNode::size(ra_); // too many variables; just compute it
 1831                               // the hard way
 1832 }
 1833 
 1834 int MachPrologNode::reloc() const
 1835 {
 1836   return 0;
 1837 }
 1838 
 1839 //=============================================================================
 1840 
 1841 #ifndef PRODUCT
 1842 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1843   Compile* C = ra_->C;
 1844   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1845 
 1846   st->print("# pop frame %d\n\t",framesize);
 1847 
 1848   if (framesize == 0) {
 1849     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1850   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1851     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1852     st->print("add  sp, sp, #%d\n\t", framesize);
 1853   } else {
 1854     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1855     st->print("add  sp, sp, rscratch1\n\t");
 1856     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1857   }
 1858   if (VM_Version::use_rop_protection()) {
 1859     st->print("autiaz\n\t");
 1860     st->print("ldr  zr, [lr]\n\t");
 1861   }
 1862 
 1863   if (do_polling() && C->is_method_compilation()) {
 1864     st->print("# test polling word\n\t");
 1865     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1866     st->print("cmp  sp, rscratch1\n\t");
 1867     st->print("bhi #slow_path");
 1868   }
 1869 }
 1870 #endif
 1871 
 1872 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1873   Compile* C = ra_->C;
 1874   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1875 
 1876   __ remove_frame(framesize);
 1877 
 1878   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1879     __ reserved_stack_check();
 1880   }
 1881 
 1882   if (do_polling() && C->is_method_compilation()) {
 1883     Label dummy_label;
 1884     Label* code_stub = &dummy_label;
 1885     if (!C->output()->in_scratch_emit_size()) {
 1886       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1887       C->output()->add_stub(stub);
 1888       code_stub = &stub->entry();
 1889     }
 1890     __ relocate(relocInfo::poll_return_type);
 1891     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1892   }
 1893 }
 1894 
 1895 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1896   // Variable size. Determine dynamically.
 1897   return MachNode::size(ra_);
 1898 }
 1899 
 1900 int MachEpilogNode::reloc() const {
 1901   // Return number of relocatable values contained in this instruction.
 1902   return 1; // 1 for polling page.
 1903 }
 1904 
 1905 const Pipeline * MachEpilogNode::pipeline() const {
 1906   return MachNode::pipeline_class();
 1907 }
 1908 
 1909 //=============================================================================
 1910 
 1911 static enum RC rc_class(OptoReg::Name reg) {
 1912 
 1913   if (reg == OptoReg::Bad) {
 1914     return rc_bad;
 1915   }
 1916 
 1917   // we have 32 int registers * 2 halves
 1918   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1919 
 1920   if (reg < slots_of_int_registers) {
 1921     return rc_int;
 1922   }
 1923 
 1924   // we have 32 float register * 8 halves
 1925   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1926   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1927     return rc_float;
 1928   }
 1929 
 1930   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1931   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1932     return rc_predicate;
 1933   }
 1934 
 1935   // Between predicate regs & stack is the flags.
 1936   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1937 
 1938   return rc_stack;
 1939 }
 1940 
 1941 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1942   Compile* C = ra_->C;
 1943 
 1944   // Get registers to move.
 1945   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1946   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1947   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1948   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1949 
 1950   enum RC src_hi_rc = rc_class(src_hi);
 1951   enum RC src_lo_rc = rc_class(src_lo);
 1952   enum RC dst_hi_rc = rc_class(dst_hi);
 1953   enum RC dst_lo_rc = rc_class(dst_lo);
 1954 
 1955   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1956 
 1957   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1958     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1959            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1960            "expected aligned-adjacent pairs");
 1961   }
 1962 
 1963   if (src_lo == dst_lo && src_hi == dst_hi) {
 1964     return 0;            // Self copy, no move.
 1965   }
 1966 
 1967   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1968               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1969   int src_offset = ra_->reg2offset(src_lo);
 1970   int dst_offset = ra_->reg2offset(dst_lo);
 1971 
 1972   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1973     uint ireg = ideal_reg();
 1974     if (ireg == Op_VecA && masm) {
 1975       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1976       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1977         // stack->stack
 1978         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1979                                                 sve_vector_reg_size_in_bytes);
 1980       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1981         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1982                             sve_vector_reg_size_in_bytes);
 1983       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1984         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1985                               sve_vector_reg_size_in_bytes);
 1986       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1987         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1988                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1989                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1990       } else {
 1991         ShouldNotReachHere();
 1992       }
 1993     } else if (masm) {
 1994       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1995       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1996       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1997         // stack->stack
 1998         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1999         if (ireg == Op_VecD) {
 2000           __ unspill(rscratch1, true, src_offset);
 2001           __ spill(rscratch1, true, dst_offset);
 2002         } else {
 2003           __ spill_copy128(src_offset, dst_offset);
 2004         }
 2005       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2006         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2007                ireg == Op_VecD ? __ T8B : __ T16B,
 2008                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2009       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2010         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2011                  ireg == Op_VecD ? __ D : __ Q,
 2012                  ra_->reg2offset(dst_lo));
 2013       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2014         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2015                    ireg == Op_VecD ? __ D : __ Q,
 2016                    ra_->reg2offset(src_lo));
 2017       } else {
 2018         ShouldNotReachHere();
 2019       }
 2020     }
 2021   } else if (masm) {
 2022     switch (src_lo_rc) {
 2023     case rc_int:
 2024       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2025         if (is64) {
 2026             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2027                    as_Register(Matcher::_regEncode[src_lo]));
 2028         } else {
 2029             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2030                     as_Register(Matcher::_regEncode[src_lo]));
 2031         }
 2032       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2033         if (is64) {
 2034             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2035                      as_Register(Matcher::_regEncode[src_lo]));
 2036         } else {
 2037             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2038                      as_Register(Matcher::_regEncode[src_lo]));
 2039         }
 2040       } else {                    // gpr --> stack spill
 2041         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2042         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2043       }
 2044       break;
 2045     case rc_float:
 2046       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2047         if (is64) {
 2048             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2049                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2050         } else {
 2051             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2052                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2053         }
 2054       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2055         if (is64) {
 2056             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2057                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2058         } else {
 2059             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2060                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2061         }
 2062       } else {                    // fpr --> stack spill
 2063         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2064         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2065                  is64 ? __ D : __ S, dst_offset);
 2066       }
 2067       break;
 2068     case rc_stack:
 2069       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2070         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2071       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2072         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2073                    is64 ? __ D : __ S, src_offset);
 2074       } else if (dst_lo_rc == rc_predicate) {
 2075         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2076                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2077       } else {                    // stack --> stack copy
 2078         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2079         if (ideal_reg() == Op_RegVectMask) {
 2080           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2081                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2082         } else {
 2083           __ unspill(rscratch1, is64, src_offset);
 2084           __ spill(rscratch1, is64, dst_offset);
 2085         }
 2086       }
 2087       break;
 2088     case rc_predicate:
 2089       if (dst_lo_rc == rc_predicate) {
 2090         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2091       } else if (dst_lo_rc == rc_stack) {
 2092         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2093                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2094       } else {
 2095         assert(false, "bad src and dst rc_class combination.");
 2096         ShouldNotReachHere();
 2097       }
 2098       break;
 2099     default:
 2100       assert(false, "bad rc_class for spill");
 2101       ShouldNotReachHere();
 2102     }
 2103   }
 2104 
 2105   if (st) {
 2106     st->print("spill ");
 2107     if (src_lo_rc == rc_stack) {
 2108       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2109     } else {
 2110       st->print("%s -> ", Matcher::regName[src_lo]);
 2111     }
 2112     if (dst_lo_rc == rc_stack) {
 2113       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2114     } else {
 2115       st->print("%s", Matcher::regName[dst_lo]);
 2116     }
 2117     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2118       int vsize = 0;
 2119       switch (ideal_reg()) {
 2120       case Op_VecD:
 2121         vsize = 64;
 2122         break;
 2123       case Op_VecX:
 2124         vsize = 128;
 2125         break;
 2126       case Op_VecA:
 2127         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2128         break;
 2129       default:
 2130         assert(false, "bad register type for spill");
 2131         ShouldNotReachHere();
 2132       }
 2133       st->print("\t# vector spill size = %d", vsize);
 2134     } else if (ideal_reg() == Op_RegVectMask) {
 2135       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2136       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2137       st->print("\t# predicate spill size = %d", vsize);
 2138     } else {
 2139       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2140     }
 2141   }
 2142 
 2143   return 0;
 2144 
 2145 }
 2146 
 2147 #ifndef PRODUCT
 2148 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2149   if (!ra_)
 2150     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2151   else
 2152     implementation(nullptr, ra_, false, st);
 2153 }
 2154 #endif
 2155 
 2156 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2157   implementation(masm, ra_, false, nullptr);
 2158 }
 2159 
 2160 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2161   return MachNode::size(ra_);
 2162 }
 2163 
 2164 //=============================================================================
 2165 
 2166 #ifndef PRODUCT
 2167 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2168   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2169   int reg = ra_->get_reg_first(this);
 2170   st->print("add %s, rsp, #%d]\t# box lock",
 2171             Matcher::regName[reg], offset);
 2172 }
 2173 #endif
 2174 
 2175 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2176   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2177   int reg    = ra_->get_encode(this);
 2178 
 2179   // This add will handle any 24-bit signed offset. 24 bits allows an
 2180   // 8 megabyte stack frame.
 2181   __ add(as_Register(reg), sp, offset);
 2182 }
 2183 
 2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2185   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2186   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2187 
 2188   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2189     return NativeInstruction::instruction_size;
 2190   } else {
 2191     return 2 * NativeInstruction::instruction_size;
 2192   }
 2193 }
 2194 
 2195 //=============================================================================
 2196 
 2197 #ifndef PRODUCT
 2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2199 {
 2200   st->print_cr("# MachUEPNode");
 2201   if (UseCompressedClassPointers) {
 2202     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2203     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2204     st->print_cr("\tcmpw rscratch1, r10");
 2205   } else {
 2206     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2207     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2208     st->print_cr("\tcmp rscratch1, r10");
 2209   }
 2210   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2211 }
 2212 #endif
 2213 
 2214 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2215 {
 2216   __ ic_check(InteriorEntryAlignment);
 2217 }
 2218 
 2219 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2220 {
 2221   return MachNode::size(ra_);
 2222 }
 2223 
 2224 // REQUIRED EMIT CODE
 2225 
 2226 //=============================================================================
 2227 
 2228 // Emit exception handler code.
 2229 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2230 {
 2231   // mov rscratch1 #exception_blob_entry_point
 2232   // br rscratch1
 2233   // Note that the code buffer's insts_mark is always relative to insts.
 2234   // That's why we must use the macroassembler to generate a handler.
 2235   address base = __ start_a_stub(size_exception_handler());
 2236   if (base == nullptr) {
 2237     ciEnv::current()->record_failure("CodeCache is full");
 2238     return 0;  // CodeBuffer::expand failed
 2239   }
 2240   int offset = __ offset();
 2241   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2242   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2243   __ end_a_stub();
 2244   return offset;
 2245 }
 2246 
 2247 // Emit deopt handler code.
 2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2249 {
 2250   // Note that the code buffer's insts_mark is always relative to insts.
 2251   // That's why we must use the macroassembler to generate a handler.
 2252   address base = __ start_a_stub(size_deopt_handler());
 2253   if (base == nullptr) {
 2254     ciEnv::current()->record_failure("CodeCache is full");
 2255     return 0;  // CodeBuffer::expand failed
 2256   }
 2257   int offset = __ offset();
 2258 
 2259   __ adr(lr, __ pc());
 2260   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2261 
 2262   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2263   __ end_a_stub();
 2264   return offset;
 2265 }
 2266 
 2267 // REQUIRED MATCHER CODE
 2268 
 2269 //=============================================================================
 2270 
 2271 bool Matcher::match_rule_supported(int opcode) {
 2272   if (!has_match_rule(opcode))
 2273     return false;
 2274 
 2275   switch (opcode) {
 2276     case Op_OnSpinWait:
 2277       return VM_Version::supports_on_spin_wait();
 2278     case Op_CacheWB:
 2279     case Op_CacheWBPreSync:
 2280     case Op_CacheWBPostSync:
 2281       if (!VM_Version::supports_data_cache_line_flush()) {
 2282         return false;
 2283       }
 2284       break;
 2285     case Op_ExpandBits:
 2286     case Op_CompressBits:
 2287       if (!VM_Version::supports_svebitperm()) {
 2288         return false;
 2289       }
 2290       break;
 2291     case Op_FmaF:
 2292     case Op_FmaD:
 2293     case Op_FmaVF:
 2294     case Op_FmaVD:
 2295       if (!UseFMA) {
 2296         return false;
 2297       }
 2298       break;
 2299     case Op_FmaHF:
 2300       // UseFMA flag also needs to be checked along with FEAT_FP16
 2301       if (!UseFMA || !is_feat_fp16_supported()) {
 2302         return false;
 2303       }
 2304       break;
 2305     case Op_AddHF:
 2306     case Op_SubHF:
 2307     case Op_MulHF:
 2308     case Op_DivHF:
 2309     case Op_MinHF:
 2310     case Op_MaxHF:
 2311     case Op_SqrtHF:
 2312       // Half-precision floating point scalar operations require FEAT_FP16
 2313       // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
 2314       // features are supported.
 2315       if (!is_feat_fp16_supported()) {
 2316         return false;
 2317       }
 2318       break;
 2319   }
 2320 
 2321   return true; // Per default match rules are supported.
 2322 }
 2323 
 2324 const RegMask* Matcher::predicate_reg_mask(void) {
 2325   return &_PR_REG_mask;
 2326 }
 2327 
 2328 bool Matcher::supports_vector_calling_convention(void) {
 2329   return EnableVectorSupport;
 2330 }
 2331 
 2332 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2333   assert(EnableVectorSupport, "sanity");
 2334   int lo = V0_num;
 2335   int hi = V0_H_num;
 2336   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2337     hi = V0_K_num;
 2338   }
 2339   return OptoRegPair(hi, lo);
 2340 }
 2341 
 2342 // Is this branch offset short enough that a short branch can be used?
 2343 //
 2344 // NOTE: If the platform does not provide any short branch variants, then
 2345 //       this method should return false for offset 0.
 2346 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2347   // The passed offset is relative to address of the branch.
 2348 
 2349   return (-32768 <= offset && offset < 32768);
 2350 }
 2351 
 2352 // Vector width in bytes.
 2353 int Matcher::vector_width_in_bytes(BasicType bt) {
 2354   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2355   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2356   // Minimum 2 values in vector
 2357   if (size < 2*type2aelembytes(bt)) size = 0;
 2358   // But never < 4
 2359   if (size < 4) size = 0;
 2360   return size;
 2361 }
 2362 
 2363 // Limits on vector size (number of elements) loaded into vector.
 2364 int Matcher::max_vector_size(const BasicType bt) {
 2365   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2366 }
 2367 
 2368 int Matcher::min_vector_size(const BasicType bt) {
 2369   int max_size = max_vector_size(bt);
 2370   // Limit the min vector size to 8 bytes.
 2371   int size = 8 / type2aelembytes(bt);
 2372   if (bt == T_BYTE) {
 2373     // To support vector api shuffle/rearrange.
 2374     size = 4;
 2375   } else if (bt == T_BOOLEAN) {
 2376     // To support vector api load/store mask.
 2377     size = 2;
 2378   }
 2379   if (size < 2) size = 2;
 2380   return MIN2(size, max_size);
 2381 }
 2382 
 2383 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2384   return Matcher::max_vector_size(bt);
 2385 }
 2386 
 2387 // Actual max scalable vector register length.
 2388 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2389   return Matcher::max_vector_size(bt);
 2390 }
 2391 
 2392 // Vector ideal reg.
 2393 uint Matcher::vector_ideal_reg(int len) {
 2394   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2395     return Op_VecA;
 2396   }
 2397   switch(len) {
 2398     // For 16-bit/32-bit mask vector, reuse VecD.
 2399     case  2:
 2400     case  4:
 2401     case  8: return Op_VecD;
 2402     case 16: return Op_VecX;
 2403   }
 2404   ShouldNotReachHere();
 2405   return 0;
 2406 }
 2407 
 2408 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2409   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2410   switch (ideal_reg) {
 2411     case Op_VecA: return new vecAOper();
 2412     case Op_VecD: return new vecDOper();
 2413     case Op_VecX: return new vecXOper();
 2414   }
 2415   ShouldNotReachHere();
 2416   return nullptr;
 2417 }
 2418 
 2419 bool Matcher::is_reg2reg_move(MachNode* m) {
 2420   return false;
 2421 }
 2422 
 2423 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2424   return opnd->opcode() == VREG;
 2425 }
 2426 
 2427 // Return whether or not this register is ever used as an argument.
 2428 // This function is used on startup to build the trampoline stubs in
 2429 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2430 // call in the trampoline, and arguments in those registers not be
 2431 // available to the callee.
 2432 bool Matcher::can_be_java_arg(int reg)
 2433 {
 2434   return
 2435     reg ==  R0_num || reg == R0_H_num ||
 2436     reg ==  R1_num || reg == R1_H_num ||
 2437     reg ==  R2_num || reg == R2_H_num ||
 2438     reg ==  R3_num || reg == R3_H_num ||
 2439     reg ==  R4_num || reg == R4_H_num ||
 2440     reg ==  R5_num || reg == R5_H_num ||
 2441     reg ==  R6_num || reg == R6_H_num ||
 2442     reg ==  R7_num || reg == R7_H_num ||
 2443     reg ==  V0_num || reg == V0_H_num ||
 2444     reg ==  V1_num || reg == V1_H_num ||
 2445     reg ==  V2_num || reg == V2_H_num ||
 2446     reg ==  V3_num || reg == V3_H_num ||
 2447     reg ==  V4_num || reg == V4_H_num ||
 2448     reg ==  V5_num || reg == V5_H_num ||
 2449     reg ==  V6_num || reg == V6_H_num ||
 2450     reg ==  V7_num || reg == V7_H_num;
 2451 }
 2452 
 2453 bool Matcher::is_spillable_arg(int reg)
 2454 {
 2455   return can_be_java_arg(reg);
 2456 }
 2457 
 2458 uint Matcher::int_pressure_limit()
 2459 {
 2460   // JDK-8183543: When taking the number of available registers as int
 2461   // register pressure threshold, the jtreg test:
 2462   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2463   // failed due to C2 compilation failure with
 2464   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2465   //
 2466   // A derived pointer is live at CallNode and then is flagged by RA
 2467   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2468   // derived pointers and lastly fail to spill after reaching maximum
 2469   // number of iterations. Lowering the default pressure threshold to
 2470   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2471   // a high register pressure area of the code so that split_DEF can
 2472   // generate DefinitionSpillCopy for the derived pointer.
 2473   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2474   if (!PreserveFramePointer) {
 2475     // When PreserveFramePointer is off, frame pointer is allocatable,
 2476     // but different from other SOC registers, it is excluded from
 2477     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2478     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2479     // See check_pressure_at_fatproj().
 2480     default_int_pressure_threshold--;
 2481   }
 2482   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2483 }
 2484 
 2485 uint Matcher::float_pressure_limit()
 2486 {
 2487   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2488   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2489 }
 2490 
 2491 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2492   return false;
 2493 }
 2494 
 2495 RegMask Matcher::divI_proj_mask() {
 2496   ShouldNotReachHere();
 2497   return RegMask();
 2498 }
 2499 
 2500 // Register for MODI projection of divmodI.
 2501 RegMask Matcher::modI_proj_mask() {
 2502   ShouldNotReachHere();
 2503   return RegMask();
 2504 }
 2505 
 2506 // Register for DIVL projection of divmodL.
 2507 RegMask Matcher::divL_proj_mask() {
 2508   ShouldNotReachHere();
 2509   return RegMask();
 2510 }
 2511 
 2512 // Register for MODL projection of divmodL.
 2513 RegMask Matcher::modL_proj_mask() {
 2514   ShouldNotReachHere();
 2515   return RegMask();
 2516 }
 2517 
 2518 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2519   return FP_REG_mask();
 2520 }
 2521 
 2522 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2523   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2524     Node* u = addp->fast_out(i);
 2525     if (u->is_LoadStore()) {
 2526       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2527       // instructions) only take register indirect as an operand, so
 2528       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2529       // must fail.
 2530       return false;
 2531     }
 2532     if (u->is_Mem()) {
 2533       int opsize = u->as_Mem()->memory_size();
 2534       assert(opsize > 0, "unexpected memory operand size");
 2535       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2536         return false;
 2537       }
 2538     }
 2539   }
 2540   return true;
 2541 }
 2542 
 2543 // Convert BootTest condition to Assembler condition.
 2544 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2545 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2546   Assembler::Condition result;
 2547   switch(cond) {
 2548     case BoolTest::eq:
 2549       result = Assembler::EQ; break;
 2550     case BoolTest::ne:
 2551       result = Assembler::NE; break;
 2552     case BoolTest::le:
 2553       result = Assembler::LE; break;
 2554     case BoolTest::ge:
 2555       result = Assembler::GE; break;
 2556     case BoolTest::lt:
 2557       result = Assembler::LT; break;
 2558     case BoolTest::gt:
 2559       result = Assembler::GT; break;
 2560     case BoolTest::ule:
 2561       result = Assembler::LS; break;
 2562     case BoolTest::uge:
 2563       result = Assembler::HS; break;
 2564     case BoolTest::ult:
 2565       result = Assembler::LO; break;
 2566     case BoolTest::ugt:
 2567       result = Assembler::HI; break;
 2568     case BoolTest::overflow:
 2569       result = Assembler::VS; break;
 2570     case BoolTest::no_overflow:
 2571       result = Assembler::VC; break;
 2572     default:
 2573       ShouldNotReachHere();
 2574       return Assembler::Condition(-1);
 2575   }
 2576 
 2577   // Check conversion
 2578   if (cond & BoolTest::unsigned_compare) {
 2579     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2580   } else {
 2581     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2582   }
 2583 
 2584   return result;
 2585 }
 2586 
 2587 // Binary src (Replicate con)
 2588 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2589   if (n == nullptr || m == nullptr) {
 2590     return false;
 2591   }
 2592 
 2593   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2594     return false;
 2595   }
 2596 
 2597   Node* imm_node = m->in(1);
 2598   if (!imm_node->is_Con()) {
 2599     return false;
 2600   }
 2601 
 2602   const Type* t = imm_node->bottom_type();
 2603   if (!(t->isa_int() || t->isa_long())) {
 2604     return false;
 2605   }
 2606 
 2607   switch (n->Opcode()) {
 2608   case Op_AndV:
 2609   case Op_OrV:
 2610   case Op_XorV: {
 2611     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2612     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2613     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2614   }
 2615   case Op_AddVB:
 2616     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2617   case Op_AddVS:
 2618   case Op_AddVI:
 2619     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2620   case Op_AddVL:
 2621     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2622   default:
 2623     return false;
 2624   }
 2625 }
 2626 
 2627 // (XorV src (Replicate m1))
 2628 // (XorVMask src (MaskAll m1))
 2629 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2630   if (n != nullptr && m != nullptr) {
 2631     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2632            VectorNode::is_all_ones_vector(m);
 2633   }
 2634   return false;
 2635 }
 2636 
 2637 // Should the matcher clone input 'm' of node 'n'?
 2638 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2639   if (is_vshift_con_pattern(n, m) ||
 2640       is_vector_bitwise_not_pattern(n, m) ||
 2641       is_valid_sve_arith_imm_pattern(n, m) ||
 2642       is_encode_and_store_pattern(n, m)) {
 2643     mstack.push(m, Visit);
 2644     return true;
 2645   }
 2646   return false;
 2647 }
 2648 
 2649 // Should the Matcher clone shifts on addressing modes, expecting them
 2650 // to be subsumed into complex addressing expressions or compute them
 2651 // into registers?
 2652 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2653 
 2654   // Loads and stores with indirect memory input (e.g., volatile loads and
 2655   // stores) do not subsume the input into complex addressing expressions. If
 2656   // the addressing expression is input to at least one such load or store, do
 2657   // not clone the addressing expression. Query needs_acquiring_load and
 2658   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2659   // possible to directly query for indirect memory input at this stage.
 2660   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2661     Node* n = m->fast_out(i);
 2662     if (n->is_Load() && needs_acquiring_load(n)) {
 2663       return false;
 2664     }
 2665     if (n->is_Store() && needs_releasing_store(n)) {
 2666       return false;
 2667     }
 2668   }
 2669 
 2670   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2671     return true;
 2672   }
 2673 
 2674   Node *off = m->in(AddPNode::Offset);
 2675   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2676       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2677       // Are there other uses besides address expressions?
 2678       !is_visited(off)) {
 2679     address_visited.set(off->_idx); // Flag as address_visited
 2680     mstack.push(off->in(2), Visit);
 2681     Node *conv = off->in(1);
 2682     if (conv->Opcode() == Op_ConvI2L &&
 2683         // Are there other uses besides address expressions?
 2684         !is_visited(conv)) {
 2685       address_visited.set(conv->_idx); // Flag as address_visited
 2686       mstack.push(conv->in(1), Pre_Visit);
 2687     } else {
 2688       mstack.push(conv, Pre_Visit);
 2689     }
 2690     address_visited.test_set(m->_idx); // Flag as address_visited
 2691     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2692     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2693     return true;
 2694   } else if (off->Opcode() == Op_ConvI2L &&
 2695              // Are there other uses besides address expressions?
 2696              !is_visited(off)) {
 2697     address_visited.test_set(m->_idx); // Flag as address_visited
 2698     address_visited.set(off->_idx); // Flag as address_visited
 2699     mstack.push(off->in(1), Pre_Visit);
 2700     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2701     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2702     return true;
 2703   }
 2704   return false;
 2705 }
 2706 
 2707 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2708   {                                                                     \
 2709     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2710     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2711     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2712     __ INSN(REG, as_Register(BASE));                                    \
 2713   }
 2714 
 2715 
 2716 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2717   {
 2718     Address::extend scale;
 2719 
 2720     // Hooboy, this is fugly.  We need a way to communicate to the
 2721     // encoder that the index needs to be sign extended, so we have to
 2722     // enumerate all the cases.
 2723     switch (opcode) {
 2724     case INDINDEXSCALEDI2L:
 2725     case INDINDEXSCALEDI2LN:
 2726     case INDINDEXI2L:
 2727     case INDINDEXI2LN:
 2728       scale = Address::sxtw(size);
 2729       break;
 2730     default:
 2731       scale = Address::lsl(size);
 2732     }
 2733 
 2734     if (index == -1) {
 2735       return Address(base, disp);
 2736     } else {
 2737       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2738       return Address(base, as_Register(index), scale);
 2739     }
 2740   }
 2741 
 2742 
 2743 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2744 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2745 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2746 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2747                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2748 
 2749   // Used for all non-volatile memory accesses.  The use of
 2750   // $mem->opcode() to discover whether this pattern uses sign-extended
 2751   // offsets is something of a kludge.
 2752   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2753                         Register reg, int opcode,
 2754                         Register base, int index, int scale, int disp,
 2755                         int size_in_memory)
 2756   {
 2757     Address addr = mem2address(opcode, base, index, scale, disp);
 2758     if (addr.getMode() == Address::base_plus_offset) {
 2759       /* Fix up any out-of-range offsets. */
 2760       assert_different_registers(rscratch1, base);
 2761       assert_different_registers(rscratch1, reg);
 2762       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2763     }
 2764     (masm->*insn)(reg, addr);
 2765   }
 2766 
 2767   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2768                         FloatRegister reg, int opcode,
 2769                         Register base, int index, int size, int disp,
 2770                         int size_in_memory)
 2771   {
 2772     Address::extend scale;
 2773 
 2774     switch (opcode) {
 2775     case INDINDEXSCALEDI2L:
 2776     case INDINDEXSCALEDI2LN:
 2777       scale = Address::sxtw(size);
 2778       break;
 2779     default:
 2780       scale = Address::lsl(size);
 2781     }
 2782 
 2783     if (index == -1) {
 2784       // Fix up any out-of-range offsets.
 2785       assert_different_registers(rscratch1, base);
 2786       Address addr = Address(base, disp);
 2787       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2788       (masm->*insn)(reg, addr);
 2789     } else {
 2790       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2791       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2792     }
 2793   }
 2794 
 2795   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2796                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2797                         int opcode, Register base, int index, int size, int disp)
 2798   {
 2799     if (index == -1) {
 2800       (masm->*insn)(reg, T, Address(base, disp));
 2801     } else {
 2802       assert(disp == 0, "unsupported address mode");
 2803       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2804     }
 2805   }
 2806 
 2807 %}
 2808 
 2809 
 2810 
 2811 //----------ENCODING BLOCK-----------------------------------------------------
 2812 // This block specifies the encoding classes used by the compiler to
 2813 // output byte streams.  Encoding classes are parameterized macros
 2814 // used by Machine Instruction Nodes in order to generate the bit
 2815 // encoding of the instruction.  Operands specify their base encoding
 2816 // interface with the interface keyword.  There are currently
 2817 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2818 // COND_INTER.  REG_INTER causes an operand to generate a function
 2819 // which returns its register number when queried.  CONST_INTER causes
 2820 // an operand to generate a function which returns the value of the
 2821 // constant when queried.  MEMORY_INTER causes an operand to generate
 2822 // four functions which return the Base Register, the Index Register,
 2823 // the Scale Value, and the Offset Value of the operand when queried.
 2824 // COND_INTER causes an operand to generate six functions which return
 2825 // the encoding code (ie - encoding bits for the instruction)
 2826 // associated with each basic boolean condition for a conditional
 2827 // instruction.
 2828 //
 2829 // Instructions specify two basic values for encoding.  Again, a
 2830 // function is available to check if the constant displacement is an
 2831 // oop. They use the ins_encode keyword to specify their encoding
 2832 // classes (which must be a sequence of enc_class names, and their
 2833 // parameters, specified in the encoding block), and they use the
 2834 // opcode keyword to specify, in order, their primary, secondary, and
 2835 // tertiary opcode.  Only the opcode sections which a particular
 2836 // instruction needs for encoding need to be specified.
 2837 encode %{
 2838   // Build emit functions for each basic byte or larger field in the
 2839   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2840   // from C++ code in the enc_class source block.  Emit functions will
 2841   // live in the main source block for now.  In future, we can
 2842   // generalize this by adding a syntax that specifies the sizes of
 2843   // fields in an order, so that the adlc can build the emit functions
 2844   // automagically
 2845 
 2846   // catch all for unimplemented encodings
 2847   enc_class enc_unimplemented %{
 2848     __ unimplemented("C2 catch all");
 2849   %}
 2850 
 2851   // BEGIN Non-volatile memory access
 2852 
 2853   // This encoding class is generated automatically from ad_encode.m4.
 2854   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2855   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2856     Register dst_reg = as_Register($dst$$reg);
 2857     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2858                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2859   %}
 2860 
 2861   // This encoding class is generated automatically from ad_encode.m4.
 2862   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2863   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2864     Register dst_reg = as_Register($dst$$reg);
 2865     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2866                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2867   %}
 2868 
 2869   // This encoding class is generated automatically from ad_encode.m4.
 2870   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2871   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2872     Register dst_reg = as_Register($dst$$reg);
 2873     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2874                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2875   %}
 2876 
 2877   // This encoding class is generated automatically from ad_encode.m4.
 2878   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2879   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2880     Register dst_reg = as_Register($dst$$reg);
 2881     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2882                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2883   %}
 2884 
 2885   // This encoding class is generated automatically from ad_encode.m4.
 2886   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2887   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2888     Register dst_reg = as_Register($dst$$reg);
 2889     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2890                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2891   %}
 2892 
 2893   // This encoding class is generated automatically from ad_encode.m4.
 2894   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2895   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2896     Register dst_reg = as_Register($dst$$reg);
 2897     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2898                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2899   %}
 2900 
 2901   // This encoding class is generated automatically from ad_encode.m4.
 2902   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2903   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2904     Register dst_reg = as_Register($dst$$reg);
 2905     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2906                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2907   %}
 2908 
 2909   // This encoding class is generated automatically from ad_encode.m4.
 2910   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2911   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2912     Register dst_reg = as_Register($dst$$reg);
 2913     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2914                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2915   %}
 2916 
 2917   // This encoding class is generated automatically from ad_encode.m4.
 2918   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2919   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2920     Register dst_reg = as_Register($dst$$reg);
 2921     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2922                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2923   %}
 2924 
 2925   // This encoding class is generated automatically from ad_encode.m4.
 2926   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2927   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2928     Register dst_reg = as_Register($dst$$reg);
 2929     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2930                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2931   %}
 2932 
 2933   // This encoding class is generated automatically from ad_encode.m4.
 2934   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2935   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2936     Register dst_reg = as_Register($dst$$reg);
 2937     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2938                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2939   %}
 2940 
 2941   // This encoding class is generated automatically from ad_encode.m4.
 2942   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2943   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2944     Register dst_reg = as_Register($dst$$reg);
 2945     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2946                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2947   %}
 2948 
 2949   // This encoding class is generated automatically from ad_encode.m4.
 2950   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2951   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2952     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2953     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2954                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2955   %}
 2956 
 2957   // This encoding class is generated automatically from ad_encode.m4.
 2958   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2959   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2960     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2961     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2962                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2963   %}
 2964 
 2965   // This encoding class is generated automatically from ad_encode.m4.
 2966   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2967   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2968     Register src_reg = as_Register($src$$reg);
 2969     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 2970                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2971   %}
 2972 
 2973   // This encoding class is generated automatically from ad_encode.m4.
 2974   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2975   enc_class aarch64_enc_strb0(memory1 mem) %{
 2976     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2977                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2978   %}
 2979 
 2980   // This encoding class is generated automatically from ad_encode.m4.
 2981   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2982   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2983     Register src_reg = as_Register($src$$reg);
 2984     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 2985                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2986   %}
 2987 
 2988   // This encoding class is generated automatically from ad_encode.m4.
 2989   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2990   enc_class aarch64_enc_strh0(memory2 mem) %{
 2991     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2992                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2993   %}
 2994 
 2995   // This encoding class is generated automatically from ad_encode.m4.
 2996   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2997   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2998     Register src_reg = as_Register($src$$reg);
 2999     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 3000                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3001   %}
 3002 
 3003   // This encoding class is generated automatically from ad_encode.m4.
 3004   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3005   enc_class aarch64_enc_strw0(memory4 mem) %{
 3006     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3013     Register src_reg = as_Register($src$$reg);
 3014     // we sometimes get asked to store the stack pointer into the
 3015     // current thread -- we cannot do that directly on AArch64
 3016     if (src_reg == r31_sp) {
 3017       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3018       __ mov(rscratch2, sp);
 3019       src_reg = rscratch2;
 3020     }
 3021     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 3022                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3023   %}
 3024 
 3025   // This encoding class is generated automatically from ad_encode.m4.
 3026   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3027   enc_class aarch64_enc_str0(memory8 mem) %{
 3028     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3029                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3030   %}
 3031 
 3032   // This encoding class is generated automatically from ad_encode.m4.
 3033   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3034   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3035     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3036     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3037                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3038   %}
 3039 
 3040   // This encoding class is generated automatically from ad_encode.m4.
 3041   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3042   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3043     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3044     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3045                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3046   %}
 3047 
 3048   // This encoding class is generated automatically from ad_encode.m4.
 3049   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3050   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3051       __ membar(Assembler::StoreStore);
 3052       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3053                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3054   %}
 3055 
 3056   // END Non-volatile memory access
 3057 
 3058   // Vector loads and stores
 3059   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3060     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3061     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3062        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3063   %}
 3064 
 3065   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3066     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3067     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3068        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3069   %}
 3070 
 3071   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3072     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3073     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3074        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3075   %}
 3076 
 3077   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3078     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3079     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3080        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3081   %}
 3082 
 3083   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3084     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3085     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3086        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3087   %}
 3088 
 3089   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3090     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3091     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3092        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3093   %}
 3094 
 3095   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3096     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3097     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3098        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3102     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3103     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3104        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3105   %}
 3106 
 3107   // volatile loads and stores
 3108 
 3109   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3110     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3111                  rscratch1, stlrb);
 3112   %}
 3113 
 3114   enc_class aarch64_enc_stlrb0(memory mem) %{
 3115     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3116                  rscratch1, stlrb);
 3117   %}
 3118 
 3119   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3120     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3121                  rscratch1, stlrh);
 3122   %}
 3123 
 3124   enc_class aarch64_enc_stlrh0(memory mem) %{
 3125     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3126                  rscratch1, stlrh);
 3127   %}
 3128 
 3129   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3130     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3131                  rscratch1, stlrw);
 3132   %}
 3133 
 3134   enc_class aarch64_enc_stlrw0(memory mem) %{
 3135     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136                  rscratch1, stlrw);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3140     Register dst_reg = as_Register($dst$$reg);
 3141     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3142              rscratch1, ldarb);
 3143     __ sxtbw(dst_reg, dst_reg);
 3144   %}
 3145 
 3146   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3147     Register dst_reg = as_Register($dst$$reg);
 3148     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3149              rscratch1, ldarb);
 3150     __ sxtb(dst_reg, dst_reg);
 3151   %}
 3152 
 3153   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3154     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3155              rscratch1, ldarb);
 3156   %}
 3157 
 3158   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3159     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160              rscratch1, ldarb);
 3161   %}
 3162 
 3163   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3164     Register dst_reg = as_Register($dst$$reg);
 3165     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarh);
 3167     __ sxthw(dst_reg, dst_reg);
 3168   %}
 3169 
 3170   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3171     Register dst_reg = as_Register($dst$$reg);
 3172     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3173              rscratch1, ldarh);
 3174     __ sxth(dst_reg, dst_reg);
 3175   %}
 3176 
 3177   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3178     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3179              rscratch1, ldarh);
 3180   %}
 3181 
 3182   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3183     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184              rscratch1, ldarh);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3188     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3189              rscratch1, ldarw);
 3190   %}
 3191 
 3192   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3193     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3194              rscratch1, ldarw);
 3195   %}
 3196 
 3197   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3198     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3199              rscratch1, ldar);
 3200   %}
 3201 
 3202   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3203     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3204              rscratch1, ldarw);
 3205     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3209     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3210              rscratch1, ldar);
 3211     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3215     Register src_reg = as_Register($src$$reg);
 3216     // we sometimes get asked to store the stack pointer into the
 3217     // current thread -- we cannot do that directly on AArch64
 3218     if (src_reg == r31_sp) {
 3219       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3220       __ mov(rscratch2, sp);
 3221       src_reg = rscratch2;
 3222     }
 3223     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3224                  rscratch1, stlr);
 3225   %}
 3226 
 3227   enc_class aarch64_enc_stlr0(memory mem) %{
 3228     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3229                  rscratch1, stlr);
 3230   %}
 3231 
 3232   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3233     {
 3234       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3235       __ fmovs(rscratch2, src_reg);
 3236     }
 3237     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3238                  rscratch1, stlrw);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3242     {
 3243       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3244       __ fmovd(rscratch2, src_reg);
 3245     }
 3246     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3247                  rscratch1, stlr);
 3248   %}
 3249 
 3250   // synchronized read/update encodings
 3251 
 3252   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3253     Register dst_reg = as_Register($dst$$reg);
 3254     Register base = as_Register($mem$$base);
 3255     int index = $mem$$index;
 3256     int scale = $mem$$scale;
 3257     int disp = $mem$$disp;
 3258     if (index == -1) {
 3259        if (disp != 0) {
 3260         __ lea(rscratch1, Address(base, disp));
 3261         __ ldaxr(dst_reg, rscratch1);
 3262       } else {
 3263         // TODO
 3264         // should we ever get anything other than this case?
 3265         __ ldaxr(dst_reg, base);
 3266       }
 3267     } else {
 3268       Register index_reg = as_Register(index);
 3269       if (disp == 0) {
 3270         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3271         __ ldaxr(dst_reg, rscratch1);
 3272       } else {
 3273         __ lea(rscratch1, Address(base, disp));
 3274         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3275         __ ldaxr(dst_reg, rscratch1);
 3276       }
 3277     }
 3278   %}
 3279 
 3280   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3281     Register src_reg = as_Register($src$$reg);
 3282     Register base = as_Register($mem$$base);
 3283     int index = $mem$$index;
 3284     int scale = $mem$$scale;
 3285     int disp = $mem$$disp;
 3286     if (index == -1) {
 3287        if (disp != 0) {
 3288         __ lea(rscratch2, Address(base, disp));
 3289         __ stlxr(rscratch1, src_reg, rscratch2);
 3290       } else {
 3291         // TODO
 3292         // should we ever get anything other than this case?
 3293         __ stlxr(rscratch1, src_reg, base);
 3294       }
 3295     } else {
 3296       Register index_reg = as_Register(index);
 3297       if (disp == 0) {
 3298         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3299         __ stlxr(rscratch1, src_reg, rscratch2);
 3300       } else {
 3301         __ lea(rscratch2, Address(base, disp));
 3302         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3303         __ stlxr(rscratch1, src_reg, rscratch2);
 3304       }
 3305     }
 3306     __ cmpw(rscratch1, zr);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3310     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3311     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3312                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3313                /*weak*/ false, noreg);
 3314   %}
 3315 
 3316   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::word, /*acquire*/ false, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3325     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3326                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3327                /*weak*/ false, noreg);
 3328   %}
 3329 
 3330   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3331     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3332     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3333                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3334                /*weak*/ false, noreg);
 3335   %}
 3336 
 3337 
 3338   // The only difference between aarch64_enc_cmpxchg and
 3339   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3340   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3341   // lock.
 3342   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3343     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3344     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3345                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3346                /*weak*/ false, noreg);
 3347   %}
 3348 
 3349   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3350     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3351     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3352                Assembler::word, /*acquire*/ true, /*release*/ true,
 3353                /*weak*/ false, noreg);
 3354   %}
 3355 
 3356   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3357     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3358     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3359                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3360                /*weak*/ false, noreg);
 3361   %}
 3362 
 3363   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3364     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3365     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3366                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3367                /*weak*/ false, noreg);
 3368   %}
 3369 
 3370   // auxiliary used for CompareAndSwapX to set result register
 3371   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3372     Register res_reg = as_Register($res$$reg);
 3373     __ cset(res_reg, Assembler::EQ);
 3374   %}
 3375 
 3376   // prefetch encodings
 3377 
 3378   enc_class aarch64_enc_prefetchw(memory mem) %{
 3379     Register base = as_Register($mem$$base);
 3380     int index = $mem$$index;
 3381     int scale = $mem$$scale;
 3382     int disp = $mem$$disp;
 3383     if (index == -1) {
 3384       // Fix up any out-of-range offsets.
 3385       assert_different_registers(rscratch1, base);
 3386       Address addr = Address(base, disp);
 3387       addr = __ legitimize_address(addr, 8, rscratch1);
 3388       __ prfm(addr, PSTL1KEEP);
 3389     } else {
 3390       Register index_reg = as_Register(index);
 3391       if (disp == 0) {
 3392         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3393       } else {
 3394         __ lea(rscratch1, Address(base, disp));
 3395 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3396       }
 3397     }
 3398   %}
 3399 
 3400   // mov encodings
 3401 
 3402   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3403     uint32_t con = (uint32_t)$src$$constant;
 3404     Register dst_reg = as_Register($dst$$reg);
 3405     if (con == 0) {
 3406       __ movw(dst_reg, zr);
 3407     } else {
 3408       __ movw(dst_reg, con);
 3409     }
 3410   %}
 3411 
 3412   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3413     Register dst_reg = as_Register($dst$$reg);
 3414     uint64_t con = (uint64_t)$src$$constant;
 3415     if (con == 0) {
 3416       __ mov(dst_reg, zr);
 3417     } else {
 3418       __ mov(dst_reg, con);
 3419     }
 3420   %}
 3421 
 3422   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3423     Register dst_reg = as_Register($dst$$reg);
 3424     address con = (address)$src$$constant;
 3425     if (con == nullptr || con == (address)1) {
 3426       ShouldNotReachHere();
 3427     } else {
 3428       relocInfo::relocType rtype = $src->constant_reloc();
 3429       if (rtype == relocInfo::oop_type) {
 3430         __ movoop(dst_reg, (jobject)con);
 3431       } else if (rtype == relocInfo::metadata_type) {
 3432         __ mov_metadata(dst_reg, (Metadata*)con);
 3433       } else {
 3434         assert(rtype == relocInfo::none, "unexpected reloc type");
 3435         if (! __ is_valid_AArch64_address(con) ||
 3436             con < (address)(uintptr_t)os::vm_page_size()) {
 3437           __ mov(dst_reg, con);
 3438         } else {
 3439           uint64_t offset;
 3440           __ adrp(dst_reg, con, offset);
 3441           __ add(dst_reg, dst_reg, offset);
 3442         }
 3443       }
 3444     }
 3445   %}
 3446 
 3447   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3448     Register dst_reg = as_Register($dst$$reg);
 3449     __ mov(dst_reg, zr);
 3450   %}
 3451 
 3452   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3453     Register dst_reg = as_Register($dst$$reg);
 3454     __ mov(dst_reg, (uint64_t)1);
 3455   %}
 3456 
 3457   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3458     __ load_byte_map_base($dst$$Register);
 3459   %}
 3460 
 3461   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3462     Register dst_reg = as_Register($dst$$reg);
 3463     address con = (address)$src$$constant;
 3464     if (con == nullptr) {
 3465       ShouldNotReachHere();
 3466     } else {
 3467       relocInfo::relocType rtype = $src->constant_reloc();
 3468       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3469       __ set_narrow_oop(dst_reg, (jobject)con);
 3470     }
 3471   %}
 3472 
 3473   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3474     Register dst_reg = as_Register($dst$$reg);
 3475     __ mov(dst_reg, zr);
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3479     Register dst_reg = as_Register($dst$$reg);
 3480     address con = (address)$src$$constant;
 3481     if (con == nullptr) {
 3482       ShouldNotReachHere();
 3483     } else {
 3484       relocInfo::relocType rtype = $src->constant_reloc();
 3485       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3486       __ set_narrow_klass(dst_reg, (Klass *)con);
 3487     }
 3488   %}
 3489 
 3490   // arithmetic encodings
 3491 
 3492   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3493     Register dst_reg = as_Register($dst$$reg);
 3494     Register src_reg = as_Register($src1$$reg);
 3495     int32_t con = (int32_t)$src2$$constant;
 3496     // add has primary == 0, subtract has primary == 1
 3497     if ($primary) { con = -con; }
 3498     if (con < 0) {
 3499       __ subw(dst_reg, src_reg, -con);
 3500     } else {
 3501       __ addw(dst_reg, src_reg, con);
 3502     }
 3503   %}
 3504 
 3505   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3506     Register dst_reg = as_Register($dst$$reg);
 3507     Register src_reg = as_Register($src1$$reg);
 3508     int32_t con = (int32_t)$src2$$constant;
 3509     // add has primary == 0, subtract has primary == 1
 3510     if ($primary) { con = -con; }
 3511     if (con < 0) {
 3512       __ sub(dst_reg, src_reg, -con);
 3513     } else {
 3514       __ add(dst_reg, src_reg, con);
 3515     }
 3516   %}
 3517 
 3518   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3519    Register dst_reg = as_Register($dst$$reg);
 3520    Register src1_reg = as_Register($src1$$reg);
 3521    Register src2_reg = as_Register($src2$$reg);
 3522     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3523   %}
 3524 
 3525   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3526    Register dst_reg = as_Register($dst$$reg);
 3527    Register src1_reg = as_Register($src1$$reg);
 3528    Register src2_reg = as_Register($src2$$reg);
 3529     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3530   %}
 3531 
 3532   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3533    Register dst_reg = as_Register($dst$$reg);
 3534    Register src1_reg = as_Register($src1$$reg);
 3535    Register src2_reg = as_Register($src2$$reg);
 3536     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3537   %}
 3538 
 3539   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3540    Register dst_reg = as_Register($dst$$reg);
 3541    Register src1_reg = as_Register($src1$$reg);
 3542    Register src2_reg = as_Register($src2$$reg);
 3543     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3544   %}
 3545 
 3546   // compare instruction encodings
 3547 
 3548   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3549     Register reg1 = as_Register($src1$$reg);
 3550     Register reg2 = as_Register($src2$$reg);
 3551     __ cmpw(reg1, reg2);
 3552   %}
 3553 
 3554   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3555     Register reg = as_Register($src1$$reg);
 3556     int32_t val = $src2$$constant;
 3557     if (val >= 0) {
 3558       __ subsw(zr, reg, val);
 3559     } else {
 3560       __ addsw(zr, reg, -val);
 3561     }
 3562   %}
 3563 
 3564   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3565     Register reg1 = as_Register($src1$$reg);
 3566     uint32_t val = (uint32_t)$src2$$constant;
 3567     __ movw(rscratch1, val);
 3568     __ cmpw(reg1, rscratch1);
 3569   %}
 3570 
 3571   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3572     Register reg1 = as_Register($src1$$reg);
 3573     Register reg2 = as_Register($src2$$reg);
 3574     __ cmp(reg1, reg2);
 3575   %}
 3576 
 3577   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3578     Register reg = as_Register($src1$$reg);
 3579     int64_t val = $src2$$constant;
 3580     if (val >= 0) {
 3581       __ subs(zr, reg, val);
 3582     } else if (val != -val) {
 3583       __ adds(zr, reg, -val);
 3584     } else {
 3585     // aargh, Long.MIN_VALUE is a special case
 3586       __ orr(rscratch1, zr, (uint64_t)val);
 3587       __ subs(zr, reg, rscratch1);
 3588     }
 3589   %}
 3590 
 3591   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3592     Register reg1 = as_Register($src1$$reg);
 3593     uint64_t val = (uint64_t)$src2$$constant;
 3594     __ mov(rscratch1, val);
 3595     __ cmp(reg1, rscratch1);
 3596   %}
 3597 
 3598   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3599     Register reg1 = as_Register($src1$$reg);
 3600     Register reg2 = as_Register($src2$$reg);
 3601     __ cmp(reg1, reg2);
 3602   %}
 3603 
 3604   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3605     Register reg1 = as_Register($src1$$reg);
 3606     Register reg2 = as_Register($src2$$reg);
 3607     __ cmpw(reg1, reg2);
 3608   %}
 3609 
 3610   enc_class aarch64_enc_testp(iRegP src) %{
 3611     Register reg = as_Register($src$$reg);
 3612     __ cmp(reg, zr);
 3613   %}
 3614 
 3615   enc_class aarch64_enc_testn(iRegN src) %{
 3616     Register reg = as_Register($src$$reg);
 3617     __ cmpw(reg, zr);
 3618   %}
 3619 
 3620   enc_class aarch64_enc_b(label lbl) %{
 3621     Label *L = $lbl$$label;
 3622     __ b(*L);
 3623   %}
 3624 
 3625   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3626     Label *L = $lbl$$label;
 3627     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3628   %}
 3629 
 3630   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3631     Label *L = $lbl$$label;
 3632     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3633   %}
 3634 
 3635   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3636   %{
 3637      Register sub_reg = as_Register($sub$$reg);
 3638      Register super_reg = as_Register($super$$reg);
 3639      Register temp_reg = as_Register($temp$$reg);
 3640      Register result_reg = as_Register($result$$reg);
 3641 
 3642      Label miss;
 3643      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3644                                      nullptr, &miss,
 3645                                      /*set_cond_codes:*/ true);
 3646      if ($primary) {
 3647        __ mov(result_reg, zr);
 3648      }
 3649      __ bind(miss);
 3650   %}
 3651 
 3652   enc_class aarch64_enc_java_static_call(method meth) %{
 3653     address addr = (address)$meth$$method;
 3654     address call;
 3655     if (!_method) {
 3656       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3657       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3658       if (call == nullptr) {
 3659         ciEnv::current()->record_failure("CodeCache is full");
 3660         return;
 3661       }
 3662     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3663       // The NOP here is purely to ensure that eliding a call to
 3664       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3665       __ nop();
 3666       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3667     } else {
 3668       int method_index = resolved_method_index(masm);
 3669       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3670                                                   : static_call_Relocation::spec(method_index);
 3671       call = __ trampoline_call(Address(addr, rspec));
 3672       if (call == nullptr) {
 3673         ciEnv::current()->record_failure("CodeCache is full");
 3674         return;
 3675       }
 3676       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3677         // Calls of the same statically bound method can share
 3678         // a stub to the interpreter.
 3679         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3680       } else {
 3681         // Emit stub for static call
 3682         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3683         if (stub == nullptr) {
 3684           ciEnv::current()->record_failure("CodeCache is full");
 3685           return;
 3686         }
 3687       }
 3688     }
 3689 
 3690     __ post_call_nop();
 3691 
 3692     // Only non uncommon_trap calls need to reinitialize ptrue.
 3693     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3694       __ reinitialize_ptrue();
 3695     }
 3696   %}
 3697 
 3698   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3699     int method_index = resolved_method_index(masm);
 3700     address call = __ ic_call((address)$meth$$method, method_index);
 3701     if (call == nullptr) {
 3702       ciEnv::current()->record_failure("CodeCache is full");
 3703       return;
 3704     }
 3705     __ post_call_nop();
 3706     if (Compile::current()->max_vector_size() > 0) {
 3707       __ reinitialize_ptrue();
 3708     }
 3709   %}
 3710 
 3711   enc_class aarch64_enc_call_epilog() %{
 3712     if (VerifyStackAtCalls) {
 3713       // Check that stack depth is unchanged: find majik cookie on stack
 3714       __ call_Unimplemented();
 3715     }
 3716   %}
 3717 
 3718   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3719     // some calls to generated routines (arraycopy code) are scheduled
 3720     // by C2 as runtime calls. if so we can call them using a br (they
 3721     // will be in a reachable segment) otherwise we have to use a blr
 3722     // which loads the absolute address into a register.
 3723     address entry = (address)$meth$$method;
 3724     CodeBlob *cb = CodeCache::find_blob(entry);
 3725     if (cb) {
 3726       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3727       if (call == nullptr) {
 3728         ciEnv::current()->record_failure("CodeCache is full");
 3729         return;
 3730       }
 3731       __ post_call_nop();
 3732     } else {
 3733       Label retaddr;
 3734       // Make the anchor frame walkable
 3735       __ adr(rscratch2, retaddr);
 3736       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3737       __ lea(rscratch1, RuntimeAddress(entry));
 3738       __ blr(rscratch1);
 3739       __ bind(retaddr);
 3740       __ post_call_nop();
 3741     }
 3742     if (Compile::current()->max_vector_size() > 0) {
 3743       __ reinitialize_ptrue();
 3744     }
 3745   %}
 3746 
 3747   enc_class aarch64_enc_rethrow() %{
 3748     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3749   %}
 3750 
 3751   enc_class aarch64_enc_ret() %{
 3752 #ifdef ASSERT
 3753     if (Compile::current()->max_vector_size() > 0) {
 3754       __ verify_ptrue();
 3755     }
 3756 #endif
 3757     __ ret(lr);
 3758   %}
 3759 
 3760   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3761     Register target_reg = as_Register($jump_target$$reg);
 3762     __ br(target_reg);
 3763   %}
 3764 
 3765   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3766     Register target_reg = as_Register($jump_target$$reg);
 3767     // exception oop should be in r0
 3768     // ret addr has been popped into lr
 3769     // callee expects it in r3
 3770     __ mov(r3, lr);
 3771     __ br(target_reg);
 3772   %}
 3773 
 3774 %}
 3775 
 3776 //----------FRAME--------------------------------------------------------------
 3777 // Definition of frame structure and management information.
 3778 //
 3779 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3780 //                             |   (to get allocators register number
 3781 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3782 //  r   CALLER     |        |
 3783 //  o     |        +--------+      pad to even-align allocators stack-slot
 3784 //  w     V        |  pad0  |        numbers; owned by CALLER
 3785 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3786 //  h     ^        |   in   |  5
 3787 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3788 //  |     |        |        |  3
 3789 //  |     |        +--------+
 3790 //  V     |        | old out|      Empty on Intel, window on Sparc
 3791 //        |    old |preserve|      Must be even aligned.
 3792 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3793 //        |        |   in   |  3   area for Intel ret address
 3794 //     Owned by    |preserve|      Empty on Sparc.
 3795 //       SELF      +--------+
 3796 //        |        |  pad2  |  2   pad to align old SP
 3797 //        |        +--------+  1
 3798 //        |        | locks  |  0
 3799 //        |        +--------+----> OptoReg::stack0(), even aligned
 3800 //        |        |  pad1  | 11   pad to align new SP
 3801 //        |        +--------+
 3802 //        |        |        | 10
 3803 //        |        | spills |  9   spills
 3804 //        V        |        |  8   (pad0 slot for callee)
 3805 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3806 //        ^        |  out   |  7
 3807 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3808 //     Owned by    +--------+
 3809 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3810 //        |    new |preserve|      Must be even-aligned.
 3811 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3812 //        |        |        |
 3813 //
 3814 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3815 //         known from SELF's arguments and the Java calling convention.
 3816 //         Region 6-7 is determined per call site.
 3817 // Note 2: If the calling convention leaves holes in the incoming argument
 3818 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3819 //         are owned by the CALLEE.  Holes should not be necessary in the
 3820 //         incoming area, as the Java calling convention is completely under
 3821 //         the control of the AD file.  Doubles can be sorted and packed to
 3822 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3823 //         varargs C calling conventions.
 3824 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3825 //         even aligned with pad0 as needed.
 3826 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3827 //           (the latter is true on Intel but is it false on AArch64?)
 3828 //         region 6-11 is even aligned; it may be padded out more so that
 3829 //         the region from SP to FP meets the minimum stack alignment.
 3830 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3831 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3832 //         SP meets the minimum alignment.
 3833 
 3834 frame %{
 3835   // These three registers define part of the calling convention
 3836   // between compiled code and the interpreter.
 3837 
 3838   // Inline Cache Register or Method for I2C.
 3839   inline_cache_reg(R12);
 3840 
 3841   // Number of stack slots consumed by locking an object
 3842   sync_stack_slots(2);
 3843 
 3844   // Compiled code's Frame Pointer
 3845   frame_pointer(R31);
 3846 
 3847   // Interpreter stores its frame pointer in a register which is
 3848   // stored to the stack by I2CAdaptors.
 3849   // I2CAdaptors convert from interpreted java to compiled java.
 3850   interpreter_frame_pointer(R29);
 3851 
 3852   // Stack alignment requirement
 3853   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3854 
 3855   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3856   // for calls to C.  Supports the var-args backing area for register parms.
 3857   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3858 
 3859   // The after-PROLOG location of the return address.  Location of
 3860   // return address specifies a type (REG or STACK) and a number
 3861   // representing the register number (i.e. - use a register name) or
 3862   // stack slot.
 3863   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3864   // Otherwise, it is above the locks and verification slot and alignment word
 3865   // TODO this may well be correct but need to check why that - 2 is there
 3866   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3867   // which folds in the space used for monitors
 3868   return_addr(STACK - 2 +
 3869               align_up((Compile::current()->in_preserve_stack_slots() +
 3870                         Compile::current()->fixed_slots()),
 3871                        stack_alignment_in_slots()));
 3872 
 3873   // Location of compiled Java return values.  Same as C for now.
 3874   return_value
 3875   %{
 3876     // TODO do we allow ideal_reg == Op_RegN???
 3877     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3878            "only return normal values");
 3879 
 3880     static const int lo[Op_RegL + 1] = { // enum name
 3881       0,                                 // Op_Node
 3882       0,                                 // Op_Set
 3883       R0_num,                            // Op_RegN
 3884       R0_num,                            // Op_RegI
 3885       R0_num,                            // Op_RegP
 3886       V0_num,                            // Op_RegF
 3887       V0_num,                            // Op_RegD
 3888       R0_num                             // Op_RegL
 3889     };
 3890 
 3891     static const int hi[Op_RegL + 1] = { // enum name
 3892       0,                                 // Op_Node
 3893       0,                                 // Op_Set
 3894       OptoReg::Bad,                      // Op_RegN
 3895       OptoReg::Bad,                      // Op_RegI
 3896       R0_H_num,                          // Op_RegP
 3897       OptoReg::Bad,                      // Op_RegF
 3898       V0_H_num,                          // Op_RegD
 3899       R0_H_num                           // Op_RegL
 3900     };
 3901 
 3902     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3903   %}
 3904 %}
 3905 
 3906 //----------ATTRIBUTES---------------------------------------------------------
 3907 //----------Operand Attributes-------------------------------------------------
 3908 op_attrib op_cost(1);        // Required cost attribute
 3909 
 3910 //----------Instruction Attributes---------------------------------------------
 3911 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3912 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3913 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3914                                 // a non-matching short branch variant
 3915                                 // of some long branch?
 3916 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3917                                 // be a power of 2) specifies the
 3918                                 // alignment that some part of the
 3919                                 // instruction (not necessarily the
 3920                                 // start) requires.  If > 1, a
 3921                                 // compute_padding() function must be
 3922                                 // provided for the instruction
 3923 
 3924 //----------OPERANDS-----------------------------------------------------------
 3925 // Operand definitions must precede instruction definitions for correct parsing
 3926 // in the ADLC because operands constitute user defined types which are used in
 3927 // instruction definitions.
 3928 
 3929 //----------Simple Operands----------------------------------------------------
 3930 
 3931 // Integer operands 32 bit
 3932 // 32 bit immediate
 3933 operand immI()
 3934 %{
 3935   match(ConI);
 3936 
 3937   op_cost(0);
 3938   format %{ %}
 3939   interface(CONST_INTER);
 3940 %}
 3941 
 3942 // 32 bit zero
 3943 operand immI0()
 3944 %{
 3945   predicate(n->get_int() == 0);
 3946   match(ConI);
 3947 
 3948   op_cost(0);
 3949   format %{ %}
 3950   interface(CONST_INTER);
 3951 %}
 3952 
 3953 // 32 bit unit increment
 3954 operand immI_1()
 3955 %{
 3956   predicate(n->get_int() == 1);
 3957   match(ConI);
 3958 
 3959   op_cost(0);
 3960   format %{ %}
 3961   interface(CONST_INTER);
 3962 %}
 3963 
 3964 // 32 bit unit decrement
 3965 operand immI_M1()
 3966 %{
 3967   predicate(n->get_int() == -1);
 3968   match(ConI);
 3969 
 3970   op_cost(0);
 3971   format %{ %}
 3972   interface(CONST_INTER);
 3973 %}
 3974 
 3975 // Shift values for add/sub extension shift
 3976 operand immIExt()
 3977 %{
 3978   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 3979   match(ConI);
 3980 
 3981   op_cost(0);
 3982   format %{ %}
 3983   interface(CONST_INTER);
 3984 %}
 3985 
 3986 operand immI_gt_1()
 3987 %{
 3988   predicate(n->get_int() > 1);
 3989   match(ConI);
 3990 
 3991   op_cost(0);
 3992   format %{ %}
 3993   interface(CONST_INTER);
 3994 %}
 3995 
 3996 operand immI_le_4()
 3997 %{
 3998   predicate(n->get_int() <= 4);
 3999   match(ConI);
 4000 
 4001   op_cost(0);
 4002   format %{ %}
 4003   interface(CONST_INTER);
 4004 %}
 4005 
 4006 operand immI_16()
 4007 %{
 4008   predicate(n->get_int() == 16);
 4009   match(ConI);
 4010 
 4011   op_cost(0);
 4012   format %{ %}
 4013   interface(CONST_INTER);
 4014 %}
 4015 
 4016 operand immI_24()
 4017 %{
 4018   predicate(n->get_int() == 24);
 4019   match(ConI);
 4020 
 4021   op_cost(0);
 4022   format %{ %}
 4023   interface(CONST_INTER);
 4024 %}
 4025 
 4026 operand immI_32()
 4027 %{
 4028   predicate(n->get_int() == 32);
 4029   match(ConI);
 4030 
 4031   op_cost(0);
 4032   format %{ %}
 4033   interface(CONST_INTER);
 4034 %}
 4035 
 4036 operand immI_48()
 4037 %{
 4038   predicate(n->get_int() == 48);
 4039   match(ConI);
 4040 
 4041   op_cost(0);
 4042   format %{ %}
 4043   interface(CONST_INTER);
 4044 %}
 4045 
 4046 operand immI_56()
 4047 %{
 4048   predicate(n->get_int() == 56);
 4049   match(ConI);
 4050 
 4051   op_cost(0);
 4052   format %{ %}
 4053   interface(CONST_INTER);
 4054 %}
 4055 
 4056 operand immI_255()
 4057 %{
 4058   predicate(n->get_int() == 255);
 4059   match(ConI);
 4060 
 4061   op_cost(0);
 4062   format %{ %}
 4063   interface(CONST_INTER);
 4064 %}
 4065 
 4066 operand immI_65535()
 4067 %{
 4068   predicate(n->get_int() == 65535);
 4069   match(ConI);
 4070 
 4071   op_cost(0);
 4072   format %{ %}
 4073   interface(CONST_INTER);
 4074 %}
 4075 
 4076 operand immI_positive()
 4077 %{
 4078   predicate(n->get_int() > 0);
 4079   match(ConI);
 4080 
 4081   op_cost(0);
 4082   format %{ %}
 4083   interface(CONST_INTER);
 4084 %}
 4085 
 4086 // BoolTest condition for signed compare
 4087 operand immI_cmp_cond()
 4088 %{
 4089   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4090   match(ConI);
 4091 
 4092   op_cost(0);
 4093   format %{ %}
 4094   interface(CONST_INTER);
 4095 %}
 4096 
 4097 // BoolTest condition for unsigned compare
 4098 operand immI_cmpU_cond()
 4099 %{
 4100   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4101   match(ConI);
 4102 
 4103   op_cost(0);
 4104   format %{ %}
 4105   interface(CONST_INTER);
 4106 %}
 4107 
 4108 operand immL_255()
 4109 %{
 4110   predicate(n->get_long() == 255L);
 4111   match(ConL);
 4112 
 4113   op_cost(0);
 4114   format %{ %}
 4115   interface(CONST_INTER);
 4116 %}
 4117 
 4118 operand immL_65535()
 4119 %{
 4120   predicate(n->get_long() == 65535L);
 4121   match(ConL);
 4122 
 4123   op_cost(0);
 4124   format %{ %}
 4125   interface(CONST_INTER);
 4126 %}
 4127 
 4128 operand immL_4294967295()
 4129 %{
 4130   predicate(n->get_long() == 4294967295L);
 4131   match(ConL);
 4132 
 4133   op_cost(0);
 4134   format %{ %}
 4135   interface(CONST_INTER);
 4136 %}
 4137 
 4138 operand immL_bitmask()
 4139 %{
 4140   predicate((n->get_long() != 0)
 4141             && ((n->get_long() & 0xc000000000000000l) == 0)
 4142             && is_power_of_2(n->get_long() + 1));
 4143   match(ConL);
 4144 
 4145   op_cost(0);
 4146   format %{ %}
 4147   interface(CONST_INTER);
 4148 %}
 4149 
 4150 operand immI_bitmask()
 4151 %{
 4152   predicate((n->get_int() != 0)
 4153             && ((n->get_int() & 0xc0000000) == 0)
 4154             && is_power_of_2(n->get_int() + 1));
 4155   match(ConI);
 4156 
 4157   op_cost(0);
 4158   format %{ %}
 4159   interface(CONST_INTER);
 4160 %}
 4161 
 4162 operand immL_positive_bitmaskI()
 4163 %{
 4164   predicate((n->get_long() != 0)
 4165             && ((julong)n->get_long() < 0x80000000ULL)
 4166             && is_power_of_2(n->get_long() + 1));
 4167   match(ConL);
 4168 
 4169   op_cost(0);
 4170   format %{ %}
 4171   interface(CONST_INTER);
 4172 %}
 4173 
 4174 // Scale values for scaled offset addressing modes (up to long but not quad)
 4175 operand immIScale()
 4176 %{
 4177   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4178   match(ConI);
 4179 
 4180   op_cost(0);
 4181   format %{ %}
 4182   interface(CONST_INTER);
 4183 %}
 4184 
 4185 // 5 bit signed integer
 4186 operand immI5()
 4187 %{
 4188   predicate(Assembler::is_simm(n->get_int(), 5));
 4189   match(ConI);
 4190 
 4191   op_cost(0);
 4192   format %{ %}
 4193   interface(CONST_INTER);
 4194 %}
 4195 
 4196 // 7 bit unsigned integer
 4197 operand immIU7()
 4198 %{
 4199   predicate(Assembler::is_uimm(n->get_int(), 7));
 4200   match(ConI);
 4201 
 4202   op_cost(0);
 4203   format %{ %}
 4204   interface(CONST_INTER);
 4205 %}
 4206 
 4207 // Offset for scaled or unscaled immediate loads and stores
 4208 operand immIOffset()
 4209 %{
 4210   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4211   match(ConI);
 4212 
 4213   op_cost(0);
 4214   format %{ %}
 4215   interface(CONST_INTER);
 4216 %}
 4217 
 4218 operand immIOffset1()
 4219 %{
 4220   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4221   match(ConI);
 4222 
 4223   op_cost(0);
 4224   format %{ %}
 4225   interface(CONST_INTER);
 4226 %}
 4227 
 4228 operand immIOffset2()
 4229 %{
 4230   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4231   match(ConI);
 4232 
 4233   op_cost(0);
 4234   format %{ %}
 4235   interface(CONST_INTER);
 4236 %}
 4237 
 4238 operand immIOffset4()
 4239 %{
 4240   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4241   match(ConI);
 4242 
 4243   op_cost(0);
 4244   format %{ %}
 4245   interface(CONST_INTER);
 4246 %}
 4247 
 4248 operand immIOffset8()
 4249 %{
 4250   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4251   match(ConI);
 4252 
 4253   op_cost(0);
 4254   format %{ %}
 4255   interface(CONST_INTER);
 4256 %}
 4257 
 4258 operand immIOffset16()
 4259 %{
 4260   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4261   match(ConI);
 4262 
 4263   op_cost(0);
 4264   format %{ %}
 4265   interface(CONST_INTER);
 4266 %}
 4267 
 4268 operand immLOffset()
 4269 %{
 4270   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4271   match(ConL);
 4272 
 4273   op_cost(0);
 4274   format %{ %}
 4275   interface(CONST_INTER);
 4276 %}
 4277 
 4278 operand immLoffset1()
 4279 %{
 4280   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4281   match(ConL);
 4282 
 4283   op_cost(0);
 4284   format %{ %}
 4285   interface(CONST_INTER);
 4286 %}
 4287 
 4288 operand immLoffset2()
 4289 %{
 4290   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4291   match(ConL);
 4292 
 4293   op_cost(0);
 4294   format %{ %}
 4295   interface(CONST_INTER);
 4296 %}
 4297 
 4298 operand immLoffset4()
 4299 %{
 4300   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4301   match(ConL);
 4302 
 4303   op_cost(0);
 4304   format %{ %}
 4305   interface(CONST_INTER);
 4306 %}
 4307 
 4308 operand immLoffset8()
 4309 %{
 4310   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4311   match(ConL);
 4312 
 4313   op_cost(0);
 4314   format %{ %}
 4315   interface(CONST_INTER);
 4316 %}
 4317 
 4318 operand immLoffset16()
 4319 %{
 4320   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4321   match(ConL);
 4322 
 4323   op_cost(0);
 4324   format %{ %}
 4325   interface(CONST_INTER);
 4326 %}
 4327 
 4328 // 5 bit signed long integer
 4329 operand immL5()
 4330 %{
 4331   predicate(Assembler::is_simm(n->get_long(), 5));
 4332   match(ConL);
 4333 
 4334   op_cost(0);
 4335   format %{ %}
 4336   interface(CONST_INTER);
 4337 %}
 4338 
 4339 // 7 bit unsigned long integer
 4340 operand immLU7()
 4341 %{
 4342   predicate(Assembler::is_uimm(n->get_long(), 7));
 4343   match(ConL);
 4344 
 4345   op_cost(0);
 4346   format %{ %}
 4347   interface(CONST_INTER);
 4348 %}
 4349 
 4350 // 8 bit signed value.
 4351 operand immI8()
 4352 %{
 4353   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4354   match(ConI);
 4355 
 4356   op_cost(0);
 4357   format %{ %}
 4358   interface(CONST_INTER);
 4359 %}
 4360 
 4361 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4362 operand immI8_shift8()
 4363 %{
 4364   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4365             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4366   match(ConI);
 4367 
 4368   op_cost(0);
 4369   format %{ %}
 4370   interface(CONST_INTER);
 4371 %}
 4372 
 4373 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4374 operand immL8_shift8()
 4375 %{
 4376   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4377             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4378   match(ConL);
 4379 
 4380   op_cost(0);
 4381   format %{ %}
 4382   interface(CONST_INTER);
 4383 %}
 4384 
 4385 // 8 bit integer valid for vector add sub immediate
 4386 operand immBAddSubV()
 4387 %{
 4388   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4389   match(ConI);
 4390 
 4391   op_cost(0);
 4392   format %{ %}
 4393   interface(CONST_INTER);
 4394 %}
 4395 
 4396 // 32 bit integer valid for add sub immediate
 4397 operand immIAddSub()
 4398 %{
 4399   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4400   match(ConI);
 4401   op_cost(0);
 4402   format %{ %}
 4403   interface(CONST_INTER);
 4404 %}
 4405 
 4406 // 32 bit integer valid for vector add sub immediate
 4407 operand immIAddSubV()
 4408 %{
 4409   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4410   match(ConI);
 4411 
 4412   op_cost(0);
 4413   format %{ %}
 4414   interface(CONST_INTER);
 4415 %}
 4416 
 4417 // 32 bit unsigned integer valid for logical immediate
 4418 
 4419 operand immBLog()
 4420 %{
 4421   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4422   match(ConI);
 4423 
 4424   op_cost(0);
 4425   format %{ %}
 4426   interface(CONST_INTER);
 4427 %}
 4428 
 4429 operand immSLog()
 4430 %{
 4431   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 operand immILog()
 4440 %{
 4441   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4442   match(ConI);
 4443 
 4444   op_cost(0);
 4445   format %{ %}
 4446   interface(CONST_INTER);
 4447 %}
 4448 
 4449 // Integer operands 64 bit
 4450 // 64 bit immediate
 4451 operand immL()
 4452 %{
 4453   match(ConL);
 4454 
 4455   op_cost(0);
 4456   format %{ %}
 4457   interface(CONST_INTER);
 4458 %}
 4459 
 4460 // 64 bit zero
 4461 operand immL0()
 4462 %{
 4463   predicate(n->get_long() == 0);
 4464   match(ConL);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 // 64 bit unit decrement
 4472 operand immL_M1()
 4473 %{
 4474   predicate(n->get_long() == -1);
 4475   match(ConL);
 4476 
 4477   op_cost(0);
 4478   format %{ %}
 4479   interface(CONST_INTER);
 4480 %}
 4481 
 4482 // 64 bit integer valid for add sub immediate
 4483 operand immLAddSub()
 4484 %{
 4485   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4486   match(ConL);
 4487   op_cost(0);
 4488   format %{ %}
 4489   interface(CONST_INTER);
 4490 %}
 4491 
 4492 // 64 bit integer valid for addv subv immediate
 4493 operand immLAddSubV()
 4494 %{
 4495   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4496   match(ConL);
 4497 
 4498   op_cost(0);
 4499   format %{ %}
 4500   interface(CONST_INTER);
 4501 %}
 4502 
 4503 // 64 bit integer valid for logical immediate
 4504 operand immLLog()
 4505 %{
 4506   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4507   match(ConL);
 4508   op_cost(0);
 4509   format %{ %}
 4510   interface(CONST_INTER);
 4511 %}
 4512 
 4513 // Long Immediate: low 32-bit mask
 4514 operand immL_32bits()
 4515 %{
 4516   predicate(n->get_long() == 0xFFFFFFFFL);
 4517   match(ConL);
 4518   op_cost(0);
 4519   format %{ %}
 4520   interface(CONST_INTER);
 4521 %}
 4522 
 4523 // Pointer operands
 4524 // Pointer Immediate
 4525 operand immP()
 4526 %{
 4527   match(ConP);
 4528 
 4529   op_cost(0);
 4530   format %{ %}
 4531   interface(CONST_INTER);
 4532 %}
 4533 
 4534 // nullptr Pointer Immediate
 4535 operand immP0()
 4536 %{
 4537   predicate(n->get_ptr() == 0);
 4538   match(ConP);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 // Pointer Immediate One
 4546 // this is used in object initialization (initial object header)
 4547 operand immP_1()
 4548 %{
 4549   predicate(n->get_ptr() == 1);
 4550   match(ConP);
 4551 
 4552   op_cost(0);
 4553   format %{ %}
 4554   interface(CONST_INTER);
 4555 %}
 4556 
 4557 // Card Table Byte Map Base
 4558 operand immByteMapBase()
 4559 %{
 4560   // Get base of card map
 4561   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4562             SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
 4563             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4564   match(ConP);
 4565 
 4566   op_cost(0);
 4567   format %{ %}
 4568   interface(CONST_INTER);
 4569 %}
 4570 
 4571 // Float and Double operands
 4572 // Double Immediate
 4573 operand immD()
 4574 %{
 4575   match(ConD);
 4576   op_cost(0);
 4577   format %{ %}
 4578   interface(CONST_INTER);
 4579 %}
 4580 
 4581 // Double Immediate: +0.0d
 4582 operand immD0()
 4583 %{
 4584   predicate(jlong_cast(n->getd()) == 0);
 4585   match(ConD);
 4586 
 4587   op_cost(0);
 4588   format %{ %}
 4589   interface(CONST_INTER);
 4590 %}
 4591 
 4592 // constant 'double +0.0'.
 4593 operand immDPacked()
 4594 %{
 4595   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4596   match(ConD);
 4597   op_cost(0);
 4598   format %{ %}
 4599   interface(CONST_INTER);
 4600 %}
 4601 
 4602 // Float Immediate
 4603 operand immF()
 4604 %{
 4605   match(ConF);
 4606   op_cost(0);
 4607   format %{ %}
 4608   interface(CONST_INTER);
 4609 %}
 4610 
 4611 // Float Immediate: +0.0f.
 4612 operand immF0()
 4613 %{
 4614   predicate(jint_cast(n->getf()) == 0);
 4615   match(ConF);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 // Half Float (FP16) Immediate
 4623 operand immH()
 4624 %{
 4625   match(ConH);
 4626   op_cost(0);
 4627   format %{ %}
 4628   interface(CONST_INTER);
 4629 %}
 4630 
 4631 //
 4632 operand immFPacked()
 4633 %{
 4634   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4635   match(ConF);
 4636   op_cost(0);
 4637   format %{ %}
 4638   interface(CONST_INTER);
 4639 %}
 4640 
 4641 // Narrow pointer operands
 4642 // Narrow Pointer Immediate
 4643 operand immN()
 4644 %{
 4645   match(ConN);
 4646 
 4647   op_cost(0);
 4648   format %{ %}
 4649   interface(CONST_INTER);
 4650 %}
 4651 
 4652 // Narrow nullptr Pointer Immediate
 4653 operand immN0()
 4654 %{
 4655   predicate(n->get_narrowcon() == 0);
 4656   match(ConN);
 4657 
 4658   op_cost(0);
 4659   format %{ %}
 4660   interface(CONST_INTER);
 4661 %}
 4662 
 4663 operand immNKlass()
 4664 %{
 4665   match(ConNKlass);
 4666 
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 // Integer 32 bit Register Operands
 4673 // Integer 32 bitRegister (excludes SP)
 4674 operand iRegI()
 4675 %{
 4676   constraint(ALLOC_IN_RC(any_reg32));
 4677   match(RegI);
 4678   match(iRegINoSp);
 4679   op_cost(0);
 4680   format %{ %}
 4681   interface(REG_INTER);
 4682 %}
 4683 
 4684 // Integer 32 bit Register not Special
 4685 operand iRegINoSp()
 4686 %{
 4687   constraint(ALLOC_IN_RC(no_special_reg32));
 4688   match(RegI);
 4689   op_cost(0);
 4690   format %{ %}
 4691   interface(REG_INTER);
 4692 %}
 4693 
 4694 // Integer 64 bit Register Operands
 4695 // Integer 64 bit Register (includes SP)
 4696 operand iRegL()
 4697 %{
 4698   constraint(ALLOC_IN_RC(any_reg));
 4699   match(RegL);
 4700   match(iRegLNoSp);
 4701   op_cost(0);
 4702   format %{ %}
 4703   interface(REG_INTER);
 4704 %}
 4705 
 4706 // Integer 64 bit Register not Special
 4707 operand iRegLNoSp()
 4708 %{
 4709   constraint(ALLOC_IN_RC(no_special_reg));
 4710   match(RegL);
 4711   match(iRegL_R0);
 4712   format %{ %}
 4713   interface(REG_INTER);
 4714 %}
 4715 
 4716 // Pointer Register Operands
 4717 // Pointer Register
 4718 operand iRegP()
 4719 %{
 4720   constraint(ALLOC_IN_RC(ptr_reg));
 4721   match(RegP);
 4722   match(iRegPNoSp);
 4723   match(iRegP_R0);
 4724   //match(iRegP_R2);
 4725   //match(iRegP_R4);
 4726   match(iRegP_R5);
 4727   match(thread_RegP);
 4728   op_cost(0);
 4729   format %{ %}
 4730   interface(REG_INTER);
 4731 %}
 4732 
 4733 // Pointer 64 bit Register not Special
 4734 operand iRegPNoSp()
 4735 %{
 4736   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4737   match(RegP);
 4738   // match(iRegP);
 4739   // match(iRegP_R0);
 4740   // match(iRegP_R2);
 4741   // match(iRegP_R4);
 4742   // match(iRegP_R5);
 4743   // match(thread_RegP);
 4744   op_cost(0);
 4745   format %{ %}
 4746   interface(REG_INTER);
 4747 %}
 4748 
 4749 // This operand is not allowed to use rfp even if
 4750 // rfp is not used to hold the frame pointer.
 4751 operand iRegPNoSpNoRfp()
 4752 %{
 4753   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4754   match(RegP);
 4755   match(iRegPNoSp);
 4756   op_cost(0);
 4757   format %{ %}
 4758   interface(REG_INTER);
 4759 %}
 4760 
 4761 // Pointer 64 bit Register R0 only
 4762 operand iRegP_R0()
 4763 %{
 4764   constraint(ALLOC_IN_RC(r0_reg));
 4765   match(RegP);
 4766   // match(iRegP);
 4767   match(iRegPNoSp);
 4768   op_cost(0);
 4769   format %{ %}
 4770   interface(REG_INTER);
 4771 %}
 4772 
 4773 // Pointer 64 bit Register R1 only
 4774 operand iRegP_R1()
 4775 %{
 4776   constraint(ALLOC_IN_RC(r1_reg));
 4777   match(RegP);
 4778   // match(iRegP);
 4779   match(iRegPNoSp);
 4780   op_cost(0);
 4781   format %{ %}
 4782   interface(REG_INTER);
 4783 %}
 4784 
 4785 // Pointer 64 bit Register R2 only
 4786 operand iRegP_R2()
 4787 %{
 4788   constraint(ALLOC_IN_RC(r2_reg));
 4789   match(RegP);
 4790   // match(iRegP);
 4791   match(iRegPNoSp);
 4792   op_cost(0);
 4793   format %{ %}
 4794   interface(REG_INTER);
 4795 %}
 4796 
 4797 // Pointer 64 bit Register R3 only
 4798 operand iRegP_R3()
 4799 %{
 4800   constraint(ALLOC_IN_RC(r3_reg));
 4801   match(RegP);
 4802   // match(iRegP);
 4803   match(iRegPNoSp);
 4804   op_cost(0);
 4805   format %{ %}
 4806   interface(REG_INTER);
 4807 %}
 4808 
 4809 // Pointer 64 bit Register R4 only
 4810 operand iRegP_R4()
 4811 %{
 4812   constraint(ALLOC_IN_RC(r4_reg));
 4813   match(RegP);
 4814   // match(iRegP);
 4815   match(iRegPNoSp);
 4816   op_cost(0);
 4817   format %{ %}
 4818   interface(REG_INTER);
 4819 %}
 4820 
 4821 // Pointer 64 bit Register R5 only
 4822 operand iRegP_R5()
 4823 %{
 4824   constraint(ALLOC_IN_RC(r5_reg));
 4825   match(RegP);
 4826   // match(iRegP);
 4827   match(iRegPNoSp);
 4828   op_cost(0);
 4829   format %{ %}
 4830   interface(REG_INTER);
 4831 %}
 4832 
 4833 // Pointer 64 bit Register R10 only
 4834 operand iRegP_R10()
 4835 %{
 4836   constraint(ALLOC_IN_RC(r10_reg));
 4837   match(RegP);
 4838   // match(iRegP);
 4839   match(iRegPNoSp);
 4840   op_cost(0);
 4841   format %{ %}
 4842   interface(REG_INTER);
 4843 %}
 4844 
 4845 // Long 64 bit Register R0 only
 4846 operand iRegL_R0()
 4847 %{
 4848   constraint(ALLOC_IN_RC(r0_reg));
 4849   match(RegL);
 4850   match(iRegLNoSp);
 4851   op_cost(0);
 4852   format %{ %}
 4853   interface(REG_INTER);
 4854 %}
 4855 
 4856 // Long 64 bit Register R11 only
 4857 operand iRegL_R11()
 4858 %{
 4859   constraint(ALLOC_IN_RC(r11_reg));
 4860   match(RegL);
 4861   match(iRegLNoSp);
 4862   op_cost(0);
 4863   format %{ %}
 4864   interface(REG_INTER);
 4865 %}
 4866 
 4867 // Register R0 only
 4868 operand iRegI_R0()
 4869 %{
 4870   constraint(ALLOC_IN_RC(int_r0_reg));
 4871   match(RegI);
 4872   match(iRegINoSp);
 4873   op_cost(0);
 4874   format %{ %}
 4875   interface(REG_INTER);
 4876 %}
 4877 
 4878 // Register R2 only
 4879 operand iRegI_R2()
 4880 %{
 4881   constraint(ALLOC_IN_RC(int_r2_reg));
 4882   match(RegI);
 4883   match(iRegINoSp);
 4884   op_cost(0);
 4885   format %{ %}
 4886   interface(REG_INTER);
 4887 %}
 4888 
 4889 // Register R3 only
 4890 operand iRegI_R3()
 4891 %{
 4892   constraint(ALLOC_IN_RC(int_r3_reg));
 4893   match(RegI);
 4894   match(iRegINoSp);
 4895   op_cost(0);
 4896   format %{ %}
 4897   interface(REG_INTER);
 4898 %}
 4899 
 4900 
 4901 // Register R4 only
 4902 operand iRegI_R4()
 4903 %{
 4904   constraint(ALLOC_IN_RC(int_r4_reg));
 4905   match(RegI);
 4906   match(iRegINoSp);
 4907   op_cost(0);
 4908   format %{ %}
 4909   interface(REG_INTER);
 4910 %}
 4911 
 4912 
 4913 // Pointer Register Operands
 4914 // Narrow Pointer Register
 4915 operand iRegN()
 4916 %{
 4917   constraint(ALLOC_IN_RC(any_reg32));
 4918   match(RegN);
 4919   match(iRegNNoSp);
 4920   op_cost(0);
 4921   format %{ %}
 4922   interface(REG_INTER);
 4923 %}
 4924 
 4925 // Integer 64 bit Register not Special
 4926 operand iRegNNoSp()
 4927 %{
 4928   constraint(ALLOC_IN_RC(no_special_reg32));
 4929   match(RegN);
 4930   op_cost(0);
 4931   format %{ %}
 4932   interface(REG_INTER);
 4933 %}
 4934 
 4935 // Float Register
 4936 // Float register operands
 4937 operand vRegF()
 4938 %{
 4939   constraint(ALLOC_IN_RC(float_reg));
 4940   match(RegF);
 4941 
 4942   op_cost(0);
 4943   format %{ %}
 4944   interface(REG_INTER);
 4945 %}
 4946 
 4947 // Double Register
 4948 // Double register operands
 4949 operand vRegD()
 4950 %{
 4951   constraint(ALLOC_IN_RC(double_reg));
 4952   match(RegD);
 4953 
 4954   op_cost(0);
 4955   format %{ %}
 4956   interface(REG_INTER);
 4957 %}
 4958 
 4959 // Generic vector class. This will be used for
 4960 // all vector operands, including NEON and SVE.
 4961 operand vReg()
 4962 %{
 4963   constraint(ALLOC_IN_RC(dynamic));
 4964   match(VecA);
 4965   match(VecD);
 4966   match(VecX);
 4967 
 4968   op_cost(0);
 4969   format %{ %}
 4970   interface(REG_INTER);
 4971 %}
 4972 
 4973 operand vecA()
 4974 %{
 4975   constraint(ALLOC_IN_RC(vectora_reg));
 4976   match(VecA);
 4977 
 4978   op_cost(0);
 4979   format %{ %}
 4980   interface(REG_INTER);
 4981 %}
 4982 
 4983 operand vecD()
 4984 %{
 4985   constraint(ALLOC_IN_RC(vectord_reg));
 4986   match(VecD);
 4987 
 4988   op_cost(0);
 4989   format %{ %}
 4990   interface(REG_INTER);
 4991 %}
 4992 
 4993 operand vecX()
 4994 %{
 4995   constraint(ALLOC_IN_RC(vectorx_reg));
 4996   match(VecX);
 4997 
 4998   op_cost(0);
 4999   format %{ %}
 5000   interface(REG_INTER);
 5001 %}
 5002 
 5003 operand vRegD_V0()
 5004 %{
 5005   constraint(ALLOC_IN_RC(v0_reg));
 5006   match(RegD);
 5007   op_cost(0);
 5008   format %{ %}
 5009   interface(REG_INTER);
 5010 %}
 5011 
 5012 operand vRegD_V1()
 5013 %{
 5014   constraint(ALLOC_IN_RC(v1_reg));
 5015   match(RegD);
 5016   op_cost(0);
 5017   format %{ %}
 5018   interface(REG_INTER);
 5019 %}
 5020 
 5021 operand vRegD_V2()
 5022 %{
 5023   constraint(ALLOC_IN_RC(v2_reg));
 5024   match(RegD);
 5025   op_cost(0);
 5026   format %{ %}
 5027   interface(REG_INTER);
 5028 %}
 5029 
 5030 operand vRegD_V3()
 5031 %{
 5032   constraint(ALLOC_IN_RC(v3_reg));
 5033   match(RegD);
 5034   op_cost(0);
 5035   format %{ %}
 5036   interface(REG_INTER);
 5037 %}
 5038 
 5039 operand vRegD_V4()
 5040 %{
 5041   constraint(ALLOC_IN_RC(v4_reg));
 5042   match(RegD);
 5043   op_cost(0);
 5044   format %{ %}
 5045   interface(REG_INTER);
 5046 %}
 5047 
 5048 operand vRegD_V5()
 5049 %{
 5050   constraint(ALLOC_IN_RC(v5_reg));
 5051   match(RegD);
 5052   op_cost(0);
 5053   format %{ %}
 5054   interface(REG_INTER);
 5055 %}
 5056 
 5057 operand vRegD_V6()
 5058 %{
 5059   constraint(ALLOC_IN_RC(v6_reg));
 5060   match(RegD);
 5061   op_cost(0);
 5062   format %{ %}
 5063   interface(REG_INTER);
 5064 %}
 5065 
 5066 operand vRegD_V7()
 5067 %{
 5068   constraint(ALLOC_IN_RC(v7_reg));
 5069   match(RegD);
 5070   op_cost(0);
 5071   format %{ %}
 5072   interface(REG_INTER);
 5073 %}
 5074 
 5075 operand vRegD_V12()
 5076 %{
 5077   constraint(ALLOC_IN_RC(v12_reg));
 5078   match(RegD);
 5079   op_cost(0);
 5080   format %{ %}
 5081   interface(REG_INTER);
 5082 %}
 5083 
 5084 operand vRegD_V13()
 5085 %{
 5086   constraint(ALLOC_IN_RC(v13_reg));
 5087   match(RegD);
 5088   op_cost(0);
 5089   format %{ %}
 5090   interface(REG_INTER);
 5091 %}
 5092 
 5093 operand pReg()
 5094 %{
 5095   constraint(ALLOC_IN_RC(pr_reg));
 5096   match(RegVectMask);
 5097   match(pRegGov);
 5098   op_cost(0);
 5099   format %{ %}
 5100   interface(REG_INTER);
 5101 %}
 5102 
 5103 operand pRegGov()
 5104 %{
 5105   constraint(ALLOC_IN_RC(gov_pr));
 5106   match(RegVectMask);
 5107   match(pReg);
 5108   op_cost(0);
 5109   format %{ %}
 5110   interface(REG_INTER);
 5111 %}
 5112 
 5113 operand pRegGov_P0()
 5114 %{
 5115   constraint(ALLOC_IN_RC(p0_reg));
 5116   match(RegVectMask);
 5117   op_cost(0);
 5118   format %{ %}
 5119   interface(REG_INTER);
 5120 %}
 5121 
 5122 operand pRegGov_P1()
 5123 %{
 5124   constraint(ALLOC_IN_RC(p1_reg));
 5125   match(RegVectMask);
 5126   op_cost(0);
 5127   format %{ %}
 5128   interface(REG_INTER);
 5129 %}
 5130 
 5131 // Flags register, used as output of signed compare instructions
 5132 
 5133 // note that on AArch64 we also use this register as the output for
 5134 // for floating point compare instructions (CmpF CmpD). this ensures
 5135 // that ordered inequality tests use GT, GE, LT or LE none of which
 5136 // pass through cases where the result is unordered i.e. one or both
 5137 // inputs to the compare is a NaN. this means that the ideal code can
 5138 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5139 // (where the comparison should always fail). EQ and NE tests are
 5140 // always generated in ideal code so that unordered folds into the NE
 5141 // case, matching the behaviour of AArch64 NE.
 5142 //
 5143 // This differs from x86 where the outputs of FP compares use a
 5144 // special FP flags registers and where compares based on this
 5145 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5146 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5147 // to explicitly handle the unordered case in branches. x86 also has
 5148 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5149 
 5150 operand rFlagsReg()
 5151 %{
 5152   constraint(ALLOC_IN_RC(int_flags));
 5153   match(RegFlags);
 5154 
 5155   op_cost(0);
 5156   format %{ "RFLAGS" %}
 5157   interface(REG_INTER);
 5158 %}
 5159 
 5160 // Flags register, used as output of unsigned compare instructions
 5161 operand rFlagsRegU()
 5162 %{
 5163   constraint(ALLOC_IN_RC(int_flags));
 5164   match(RegFlags);
 5165 
 5166   op_cost(0);
 5167   format %{ "RFLAGSU" %}
 5168   interface(REG_INTER);
 5169 %}
 5170 
 5171 // Special Registers
 5172 
 5173 // Method Register
 5174 operand inline_cache_RegP(iRegP reg)
 5175 %{
 5176   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5177   match(reg);
 5178   match(iRegPNoSp);
 5179   op_cost(0);
 5180   format %{ %}
 5181   interface(REG_INTER);
 5182 %}
 5183 
 5184 // Thread Register
 5185 operand thread_RegP(iRegP reg)
 5186 %{
 5187   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5188   match(reg);
 5189   op_cost(0);
 5190   format %{ %}
 5191   interface(REG_INTER);
 5192 %}
 5193 
 5194 //----------Memory Operands----------------------------------------------------
 5195 
 5196 operand indirect(iRegP reg)
 5197 %{
 5198   constraint(ALLOC_IN_RC(ptr_reg));
 5199   match(reg);
 5200   op_cost(0);
 5201   format %{ "[$reg]" %}
 5202   interface(MEMORY_INTER) %{
 5203     base($reg);
 5204     index(0xffffffff);
 5205     scale(0x0);
 5206     disp(0x0);
 5207   %}
 5208 %}
 5209 
 5210 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5211 %{
 5212   constraint(ALLOC_IN_RC(ptr_reg));
 5213   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5214   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5215   op_cost(0);
 5216   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5217   interface(MEMORY_INTER) %{
 5218     base($reg);
 5219     index($ireg);
 5220     scale($scale);
 5221     disp(0x0);
 5222   %}
 5223 %}
 5224 
 5225 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5226 %{
 5227   constraint(ALLOC_IN_RC(ptr_reg));
 5228   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5229   match(AddP reg (LShiftL lreg scale));
 5230   op_cost(0);
 5231   format %{ "$reg, $lreg lsl($scale)" %}
 5232   interface(MEMORY_INTER) %{
 5233     base($reg);
 5234     index($lreg);
 5235     scale($scale);
 5236     disp(0x0);
 5237   %}
 5238 %}
 5239 
 5240 operand indIndexI2L(iRegP reg, iRegI ireg)
 5241 %{
 5242   constraint(ALLOC_IN_RC(ptr_reg));
 5243   match(AddP reg (ConvI2L ireg));
 5244   op_cost(0);
 5245   format %{ "$reg, $ireg, 0, I2L" %}
 5246   interface(MEMORY_INTER) %{
 5247     base($reg);
 5248     index($ireg);
 5249     scale(0x0);
 5250     disp(0x0);
 5251   %}
 5252 %}
 5253 
 5254 operand indIndex(iRegP reg, iRegL lreg)
 5255 %{
 5256   constraint(ALLOC_IN_RC(ptr_reg));
 5257   match(AddP reg lreg);
 5258   op_cost(0);
 5259   format %{ "$reg, $lreg" %}
 5260   interface(MEMORY_INTER) %{
 5261     base($reg);
 5262     index($lreg);
 5263     scale(0x0);
 5264     disp(0x0);
 5265   %}
 5266 %}
 5267 
 5268 operand indOffI1(iRegP reg, immIOffset1 off)
 5269 %{
 5270   constraint(ALLOC_IN_RC(ptr_reg));
 5271   match(AddP reg off);
 5272   op_cost(0);
 5273   format %{ "[$reg, $off]" %}
 5274   interface(MEMORY_INTER) %{
 5275     base($reg);
 5276     index(0xffffffff);
 5277     scale(0x0);
 5278     disp($off);
 5279   %}
 5280 %}
 5281 
 5282 operand indOffI2(iRegP reg, immIOffset2 off)
 5283 %{
 5284   constraint(ALLOC_IN_RC(ptr_reg));
 5285   match(AddP reg off);
 5286   op_cost(0);
 5287   format %{ "[$reg, $off]" %}
 5288   interface(MEMORY_INTER) %{
 5289     base($reg);
 5290     index(0xffffffff);
 5291     scale(0x0);
 5292     disp($off);
 5293   %}
 5294 %}
 5295 
 5296 operand indOffI4(iRegP reg, immIOffset4 off)
 5297 %{
 5298   constraint(ALLOC_IN_RC(ptr_reg));
 5299   match(AddP reg off);
 5300   op_cost(0);
 5301   format %{ "[$reg, $off]" %}
 5302   interface(MEMORY_INTER) %{
 5303     base($reg);
 5304     index(0xffffffff);
 5305     scale(0x0);
 5306     disp($off);
 5307   %}
 5308 %}
 5309 
 5310 operand indOffI8(iRegP reg, immIOffset8 off)
 5311 %{
 5312   constraint(ALLOC_IN_RC(ptr_reg));
 5313   match(AddP reg off);
 5314   op_cost(0);
 5315   format %{ "[$reg, $off]" %}
 5316   interface(MEMORY_INTER) %{
 5317     base($reg);
 5318     index(0xffffffff);
 5319     scale(0x0);
 5320     disp($off);
 5321   %}
 5322 %}
 5323 
 5324 operand indOffI16(iRegP reg, immIOffset16 off)
 5325 %{
 5326   constraint(ALLOC_IN_RC(ptr_reg));
 5327   match(AddP reg off);
 5328   op_cost(0);
 5329   format %{ "[$reg, $off]" %}
 5330   interface(MEMORY_INTER) %{
 5331     base($reg);
 5332     index(0xffffffff);
 5333     scale(0x0);
 5334     disp($off);
 5335   %}
 5336 %}
 5337 
 5338 operand indOffL1(iRegP reg, immLoffset1 off)
 5339 %{
 5340   constraint(ALLOC_IN_RC(ptr_reg));
 5341   match(AddP reg off);
 5342   op_cost(0);
 5343   format %{ "[$reg, $off]" %}
 5344   interface(MEMORY_INTER) %{
 5345     base($reg);
 5346     index(0xffffffff);
 5347     scale(0x0);
 5348     disp($off);
 5349   %}
 5350 %}
 5351 
 5352 operand indOffL2(iRegP reg, immLoffset2 off)
 5353 %{
 5354   constraint(ALLOC_IN_RC(ptr_reg));
 5355   match(AddP reg off);
 5356   op_cost(0);
 5357   format %{ "[$reg, $off]" %}
 5358   interface(MEMORY_INTER) %{
 5359     base($reg);
 5360     index(0xffffffff);
 5361     scale(0x0);
 5362     disp($off);
 5363   %}
 5364 %}
 5365 
 5366 operand indOffL4(iRegP reg, immLoffset4 off)
 5367 %{
 5368   constraint(ALLOC_IN_RC(ptr_reg));
 5369   match(AddP reg off);
 5370   op_cost(0);
 5371   format %{ "[$reg, $off]" %}
 5372   interface(MEMORY_INTER) %{
 5373     base($reg);
 5374     index(0xffffffff);
 5375     scale(0x0);
 5376     disp($off);
 5377   %}
 5378 %}
 5379 
 5380 operand indOffL8(iRegP reg, immLoffset8 off)
 5381 %{
 5382   constraint(ALLOC_IN_RC(ptr_reg));
 5383   match(AddP reg off);
 5384   op_cost(0);
 5385   format %{ "[$reg, $off]" %}
 5386   interface(MEMORY_INTER) %{
 5387     base($reg);
 5388     index(0xffffffff);
 5389     scale(0x0);
 5390     disp($off);
 5391   %}
 5392 %}
 5393 
 5394 operand indOffL16(iRegP reg, immLoffset16 off)
 5395 %{
 5396   constraint(ALLOC_IN_RC(ptr_reg));
 5397   match(AddP reg off);
 5398   op_cost(0);
 5399   format %{ "[$reg, $off]" %}
 5400   interface(MEMORY_INTER) %{
 5401     base($reg);
 5402     index(0xffffffff);
 5403     scale(0x0);
 5404     disp($off);
 5405   %}
 5406 %}
 5407 
 5408 operand indirectX2P(iRegL reg)
 5409 %{
 5410   constraint(ALLOC_IN_RC(ptr_reg));
 5411   match(CastX2P reg);
 5412   op_cost(0);
 5413   format %{ "[$reg]\t# long -> ptr" %}
 5414   interface(MEMORY_INTER) %{
 5415     base($reg);
 5416     index(0xffffffff);
 5417     scale(0x0);
 5418     disp(0x0);
 5419   %}
 5420 %}
 5421 
 5422 operand indOffX2P(iRegL reg, immLOffset off)
 5423 %{
 5424   constraint(ALLOC_IN_RC(ptr_reg));
 5425   match(AddP (CastX2P reg) off);
 5426   op_cost(0);
 5427   format %{ "[$reg, $off]\t# long -> ptr" %}
 5428   interface(MEMORY_INTER) %{
 5429     base($reg);
 5430     index(0xffffffff);
 5431     scale(0x0);
 5432     disp($off);
 5433   %}
 5434 %}
 5435 
 5436 operand indirectN(iRegN reg)
 5437 %{
 5438   predicate(CompressedOops::shift() == 0);
 5439   constraint(ALLOC_IN_RC(ptr_reg));
 5440   match(DecodeN reg);
 5441   op_cost(0);
 5442   format %{ "[$reg]\t# narrow" %}
 5443   interface(MEMORY_INTER) %{
 5444     base($reg);
 5445     index(0xffffffff);
 5446     scale(0x0);
 5447     disp(0x0);
 5448   %}
 5449 %}
 5450 
 5451 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5452 %{
 5453   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5454   constraint(ALLOC_IN_RC(ptr_reg));
 5455   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5456   op_cost(0);
 5457   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5458   interface(MEMORY_INTER) %{
 5459     base($reg);
 5460     index($ireg);
 5461     scale($scale);
 5462     disp(0x0);
 5463   %}
 5464 %}
 5465 
 5466 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5467 %{
 5468   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5469   constraint(ALLOC_IN_RC(ptr_reg));
 5470   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5471   op_cost(0);
 5472   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5473   interface(MEMORY_INTER) %{
 5474     base($reg);
 5475     index($lreg);
 5476     scale($scale);
 5477     disp(0x0);
 5478   %}
 5479 %}
 5480 
 5481 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5482 %{
 5483   predicate(CompressedOops::shift() == 0);
 5484   constraint(ALLOC_IN_RC(ptr_reg));
 5485   match(AddP (DecodeN reg) (ConvI2L ireg));
 5486   op_cost(0);
 5487   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5488   interface(MEMORY_INTER) %{
 5489     base($reg);
 5490     index($ireg);
 5491     scale(0x0);
 5492     disp(0x0);
 5493   %}
 5494 %}
 5495 
 5496 operand indIndexN(iRegN reg, iRegL lreg)
 5497 %{
 5498   predicate(CompressedOops::shift() == 0);
 5499   constraint(ALLOC_IN_RC(ptr_reg));
 5500   match(AddP (DecodeN reg) lreg);
 5501   op_cost(0);
 5502   format %{ "$reg, $lreg\t# narrow" %}
 5503   interface(MEMORY_INTER) %{
 5504     base($reg);
 5505     index($lreg);
 5506     scale(0x0);
 5507     disp(0x0);
 5508   %}
 5509 %}
 5510 
 5511 operand indOffIN(iRegN reg, immIOffset off)
 5512 %{
 5513   predicate(CompressedOops::shift() == 0);
 5514   constraint(ALLOC_IN_RC(ptr_reg));
 5515   match(AddP (DecodeN reg) off);
 5516   op_cost(0);
 5517   format %{ "[$reg, $off]\t# narrow" %}
 5518   interface(MEMORY_INTER) %{
 5519     base($reg);
 5520     index(0xffffffff);
 5521     scale(0x0);
 5522     disp($off);
 5523   %}
 5524 %}
 5525 
 5526 operand indOffLN(iRegN reg, immLOffset off)
 5527 %{
 5528   predicate(CompressedOops::shift() == 0);
 5529   constraint(ALLOC_IN_RC(ptr_reg));
 5530   match(AddP (DecodeN reg) off);
 5531   op_cost(0);
 5532   format %{ "[$reg, $off]\t# narrow" %}
 5533   interface(MEMORY_INTER) %{
 5534     base($reg);
 5535     index(0xffffffff);
 5536     scale(0x0);
 5537     disp($off);
 5538   %}
 5539 %}
 5540 
 5541 
 5542 //----------Special Memory Operands--------------------------------------------
 5543 // Stack Slot Operand - This operand is used for loading and storing temporary
 5544 //                      values on the stack where a match requires a value to
 5545 //                      flow through memory.
 5546 operand stackSlotP(sRegP reg)
 5547 %{
 5548   constraint(ALLOC_IN_RC(stack_slots));
 5549   op_cost(100);
 5550   // No match rule because this operand is only generated in matching
 5551   // match(RegP);
 5552   format %{ "[$reg]" %}
 5553   interface(MEMORY_INTER) %{
 5554     base(0x1e);  // RSP
 5555     index(0x0);  // No Index
 5556     scale(0x0);  // No Scale
 5557     disp($reg);  // Stack Offset
 5558   %}
 5559 %}
 5560 
 5561 operand stackSlotI(sRegI reg)
 5562 %{
 5563   constraint(ALLOC_IN_RC(stack_slots));
 5564   // No match rule because this operand is only generated in matching
 5565   // match(RegI);
 5566   format %{ "[$reg]" %}
 5567   interface(MEMORY_INTER) %{
 5568     base(0x1e);  // RSP
 5569     index(0x0);  // No Index
 5570     scale(0x0);  // No Scale
 5571     disp($reg);  // Stack Offset
 5572   %}
 5573 %}
 5574 
 5575 operand stackSlotF(sRegF reg)
 5576 %{
 5577   constraint(ALLOC_IN_RC(stack_slots));
 5578   // No match rule because this operand is only generated in matching
 5579   // match(RegF);
 5580   format %{ "[$reg]" %}
 5581   interface(MEMORY_INTER) %{
 5582     base(0x1e);  // RSP
 5583     index(0x0);  // No Index
 5584     scale(0x0);  // No Scale
 5585     disp($reg);  // Stack Offset
 5586   %}
 5587 %}
 5588 
 5589 operand stackSlotD(sRegD reg)
 5590 %{
 5591   constraint(ALLOC_IN_RC(stack_slots));
 5592   // No match rule because this operand is only generated in matching
 5593   // match(RegD);
 5594   format %{ "[$reg]" %}
 5595   interface(MEMORY_INTER) %{
 5596     base(0x1e);  // RSP
 5597     index(0x0);  // No Index
 5598     scale(0x0);  // No Scale
 5599     disp($reg);  // Stack Offset
 5600   %}
 5601 %}
 5602 
 5603 operand stackSlotL(sRegL reg)
 5604 %{
 5605   constraint(ALLOC_IN_RC(stack_slots));
 5606   // No match rule because this operand is only generated in matching
 5607   // match(RegL);
 5608   format %{ "[$reg]" %}
 5609   interface(MEMORY_INTER) %{
 5610     base(0x1e);  // RSP
 5611     index(0x0);  // No Index
 5612     scale(0x0);  // No Scale
 5613     disp($reg);  // Stack Offset
 5614   %}
 5615 %}
 5616 
 5617 // Operands for expressing Control Flow
 5618 // NOTE: Label is a predefined operand which should not be redefined in
 5619 //       the AD file. It is generically handled within the ADLC.
 5620 
 5621 //----------Conditional Branch Operands----------------------------------------
 5622 // Comparison Op  - This is the operation of the comparison, and is limited to
 5623 //                  the following set of codes:
 5624 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5625 //
 5626 // Other attributes of the comparison, such as unsignedness, are specified
 5627 // by the comparison instruction that sets a condition code flags register.
 5628 // That result is represented by a flags operand whose subtype is appropriate
 5629 // to the unsignedness (etc.) of the comparison.
 5630 //
 5631 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5632 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5633 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5634 
 5635 // used for signed integral comparisons and fp comparisons
 5636 
 5637 operand cmpOp()
 5638 %{
 5639   match(Bool);
 5640 
 5641   format %{ "" %}
 5642   interface(COND_INTER) %{
 5643     equal(0x0, "eq");
 5644     not_equal(0x1, "ne");
 5645     less(0xb, "lt");
 5646     greater_equal(0xa, "ge");
 5647     less_equal(0xd, "le");
 5648     greater(0xc, "gt");
 5649     overflow(0x6, "vs");
 5650     no_overflow(0x7, "vc");
 5651   %}
 5652 %}
 5653 
 5654 // used for unsigned integral comparisons
 5655 
 5656 operand cmpOpU()
 5657 %{
 5658   match(Bool);
 5659 
 5660   format %{ "" %}
 5661   interface(COND_INTER) %{
 5662     equal(0x0, "eq");
 5663     not_equal(0x1, "ne");
 5664     less(0x3, "lo");
 5665     greater_equal(0x2, "hs");
 5666     less_equal(0x9, "ls");
 5667     greater(0x8, "hi");
 5668     overflow(0x6, "vs");
 5669     no_overflow(0x7, "vc");
 5670   %}
 5671 %}
 5672 
 5673 // used for certain integral comparisons which can be
 5674 // converted to cbxx or tbxx instructions
 5675 
 5676 operand cmpOpEqNe()
 5677 %{
 5678   match(Bool);
 5679   op_cost(0);
 5680   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5681             || n->as_Bool()->_test._test == BoolTest::eq);
 5682 
 5683   format %{ "" %}
 5684   interface(COND_INTER) %{
 5685     equal(0x0, "eq");
 5686     not_equal(0x1, "ne");
 5687     less(0xb, "lt");
 5688     greater_equal(0xa, "ge");
 5689     less_equal(0xd, "le");
 5690     greater(0xc, "gt");
 5691     overflow(0x6, "vs");
 5692     no_overflow(0x7, "vc");
 5693   %}
 5694 %}
 5695 
 5696 // used for certain integral comparisons which can be
 5697 // converted to cbxx or tbxx instructions
 5698 
 5699 operand cmpOpLtGe()
 5700 %{
 5701   match(Bool);
 5702   op_cost(0);
 5703 
 5704   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5705             || n->as_Bool()->_test._test == BoolTest::ge);
 5706 
 5707   format %{ "" %}
 5708   interface(COND_INTER) %{
 5709     equal(0x0, "eq");
 5710     not_equal(0x1, "ne");
 5711     less(0xb, "lt");
 5712     greater_equal(0xa, "ge");
 5713     less_equal(0xd, "le");
 5714     greater(0xc, "gt");
 5715     overflow(0x6, "vs");
 5716     no_overflow(0x7, "vc");
 5717   %}
 5718 %}
 5719 
 5720 // used for certain unsigned integral comparisons which can be
 5721 // converted to cbxx or tbxx instructions
 5722 
 5723 operand cmpOpUEqNeLeGt()
 5724 %{
 5725   match(Bool);
 5726   op_cost(0);
 5727 
 5728   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5729             n->as_Bool()->_test._test == BoolTest::ne ||
 5730             n->as_Bool()->_test._test == BoolTest::le ||
 5731             n->as_Bool()->_test._test == BoolTest::gt);
 5732 
 5733   format %{ "" %}
 5734   interface(COND_INTER) %{
 5735     equal(0x0, "eq");
 5736     not_equal(0x1, "ne");
 5737     less(0x3, "lo");
 5738     greater_equal(0x2, "hs");
 5739     less_equal(0x9, "ls");
 5740     greater(0x8, "hi");
 5741     overflow(0x6, "vs");
 5742     no_overflow(0x7, "vc");
 5743   %}
 5744 %}
 5745 
 5746 // Special operand allowing long args to int ops to be truncated for free
 5747 
 5748 operand iRegL2I(iRegL reg) %{
 5749 
 5750   op_cost(0);
 5751 
 5752   match(ConvL2I reg);
 5753 
 5754   format %{ "l2i($reg)" %}
 5755 
 5756   interface(REG_INTER)
 5757 %}
 5758 
 5759 operand iRegL2P(iRegL reg) %{
 5760 
 5761   op_cost(0);
 5762 
 5763   match(CastX2P reg);
 5764 
 5765   format %{ "l2p($reg)" %}
 5766 
 5767   interface(REG_INTER)
 5768 %}
 5769 
 5770 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5771 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5772 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5773 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5774 
 5775 //----------OPERAND CLASSES----------------------------------------------------
 5776 // Operand Classes are groups of operands that are used as to simplify
 5777 // instruction definitions by not requiring the AD writer to specify
 5778 // separate instructions for every form of operand when the
 5779 // instruction accepts multiple operand types with the same basic
 5780 // encoding and format. The classic case of this is memory operands.
 5781 
 5782 // memory is used to define read/write location for load/store
 5783 // instruction defs. we can turn a memory op into an Address
 5784 
 5785 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5786                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5787 
 5788 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5789                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5790 
 5791 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5792                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5793 
 5794 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5795                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5796 
 5797 // All of the memory operands. For the pipeline description.
 5798 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5799                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5800                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5801 
 5802 opclass memory_noindex(indirect,
 5803                        indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5804                        indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5805 
 5806 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5807 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5808 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5809 // can be elided because the 32-bit instruction will just employ the
 5810 // lower 32 bits anyway.
 5811 //
 5812 // n.b. this does not elide all L2I conversions. if the truncated
 5813 // value is consumed by more than one operation then the ConvL2I
 5814 // cannot be bundled into the consuming nodes so an l2i gets planted
 5815 // (actually a movw $dst $src) and the downstream instructions consume
 5816 // the result of the l2i as an iRegI input. That's a shame since the
 5817 // movw is actually redundant but its not too costly.
 5818 
 5819 opclass iRegIorL2I(iRegI, iRegL2I);
 5820 opclass iRegPorL2P(iRegP, iRegL2P);
 5821 
 5822 //----------PIPELINE-----------------------------------------------------------
 5823 // Rules which define the behavior of the target architectures pipeline.
 5824 
 5825 // For specific pipelines, eg A53, define the stages of that pipeline
 5826 //pipe_desc(ISS, EX1, EX2, WR);
 5827 #define ISS S0
 5828 #define EX1 S1
 5829 #define EX2 S2
 5830 #define WR  S3
 5831 
 5832 // Integer ALU reg operation
 5833 pipeline %{
 5834 
 5835 attributes %{
 5836   // ARM instructions are of fixed length
 5837   fixed_size_instructions;        // Fixed size instructions TODO does
 5838   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5839   // ARM instructions come in 32-bit word units
 5840   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5841   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5842   instruction_fetch_units = 1;       // of 64 bytes
 5843 
 5844   // List of nop instructions
 5845   nops( MachNop );
 5846 %}
 5847 
 5848 // We don't use an actual pipeline model so don't care about resources
 5849 // or description. we do use pipeline classes to introduce fixed
 5850 // latencies
 5851 
 5852 //----------RESOURCES----------------------------------------------------------
 5853 // Resources are the functional units available to the machine
 5854 
 5855 resources( INS0, INS1, INS01 = INS0 | INS1,
 5856            ALU0, ALU1, ALU = ALU0 | ALU1,
 5857            MAC,
 5858            DIV,
 5859            BRANCH,
 5860            LDST,
 5861            NEON_FP);
 5862 
 5863 //----------PIPELINE DESCRIPTION-----------------------------------------------
 5864 // Pipeline Description specifies the stages in the machine's pipeline
 5865 
 5866 // Define the pipeline as a generic 6 stage pipeline
 5867 pipe_desc(S0, S1, S2, S3, S4, S5);
 5868 
 5869 //----------PIPELINE CLASSES---------------------------------------------------
 5870 // Pipeline Classes describe the stages in which input and output are
 5871 // referenced by the hardware pipeline.
 5872 
 5873 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 5874 %{
 5875   single_instruction;
 5876   src1   : S1(read);
 5877   src2   : S2(read);
 5878   dst    : S5(write);
 5879   INS01  : ISS;
 5880   NEON_FP : S5;
 5881 %}
 5882 
 5883 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 5884 %{
 5885   single_instruction;
 5886   src1   : S1(read);
 5887   src2   : S2(read);
 5888   dst    : S5(write);
 5889   INS01  : ISS;
 5890   NEON_FP : S5;
 5891 %}
 5892 
 5893 pipe_class fp_uop_s(vRegF dst, vRegF src)
 5894 %{
 5895   single_instruction;
 5896   src    : S1(read);
 5897   dst    : S5(write);
 5898   INS01  : ISS;
 5899   NEON_FP : S5;
 5900 %}
 5901 
 5902 pipe_class fp_uop_d(vRegD dst, vRegD src)
 5903 %{
 5904   single_instruction;
 5905   src    : S1(read);
 5906   dst    : S5(write);
 5907   INS01  : ISS;
 5908   NEON_FP : S5;
 5909 %}
 5910 
 5911 pipe_class fp_d2f(vRegF dst, vRegD src)
 5912 %{
 5913   single_instruction;
 5914   src    : S1(read);
 5915   dst    : S5(write);
 5916   INS01  : ISS;
 5917   NEON_FP : S5;
 5918 %}
 5919 
 5920 pipe_class fp_f2d(vRegD dst, vRegF src)
 5921 %{
 5922   single_instruction;
 5923   src    : S1(read);
 5924   dst    : S5(write);
 5925   INS01  : ISS;
 5926   NEON_FP : S5;
 5927 %}
 5928 
 5929 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 5930 %{
 5931   single_instruction;
 5932   src    : S1(read);
 5933   dst    : S5(write);
 5934   INS01  : ISS;
 5935   NEON_FP : S5;
 5936 %}
 5937 
 5938 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 5939 %{
 5940   single_instruction;
 5941   src    : S1(read);
 5942   dst    : S5(write);
 5943   INS01  : ISS;
 5944   NEON_FP : S5;
 5945 %}
 5946 
 5947 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 5948 %{
 5949   single_instruction;
 5950   src    : S1(read);
 5951   dst    : S5(write);
 5952   INS01  : ISS;
 5953   NEON_FP : S5;
 5954 %}
 5955 
 5956 pipe_class fp_l2f(vRegF dst, iRegL src)
 5957 %{
 5958   single_instruction;
 5959   src    : S1(read);
 5960   dst    : S5(write);
 5961   INS01  : ISS;
 5962   NEON_FP : S5;
 5963 %}
 5964 
 5965 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 5966 %{
 5967   single_instruction;
 5968   src    : S1(read);
 5969   dst    : S5(write);
 5970   INS01  : ISS;
 5971   NEON_FP : S5;
 5972 %}
 5973 
 5974 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 5975 %{
 5976   single_instruction;
 5977   src    : S1(read);
 5978   dst    : S5(write);
 5979   INS01  : ISS;
 5980   NEON_FP : S5;
 5981 %}
 5982 
 5983 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 5984 %{
 5985   single_instruction;
 5986   src    : S1(read);
 5987   dst    : S5(write);
 5988   INS01  : ISS;
 5989   NEON_FP : S5;
 5990 %}
 5991 
 5992 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 5993 %{
 5994   single_instruction;
 5995   src    : S1(read);
 5996   dst    : S5(write);
 5997   INS01  : ISS;
 5998   NEON_FP : S5;
 5999 %}
 6000 
 6001 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6002 %{
 6003   single_instruction;
 6004   src1   : S1(read);
 6005   src2   : S2(read);
 6006   dst    : S5(write);
 6007   INS0   : ISS;
 6008   NEON_FP : S5;
 6009 %}
 6010 
 6011 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6012 %{
 6013   single_instruction;
 6014   src1   : S1(read);
 6015   src2   : S2(read);
 6016   dst    : S5(write);
 6017   INS0   : ISS;
 6018   NEON_FP : S5;
 6019 %}
 6020 
 6021 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6022 %{
 6023   single_instruction;
 6024   cr     : S1(read);
 6025   src1   : S1(read);
 6026   src2   : S1(read);
 6027   dst    : S3(write);
 6028   INS01  : ISS;
 6029   NEON_FP : S3;
 6030 %}
 6031 
 6032 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6033 %{
 6034   single_instruction;
 6035   cr     : S1(read);
 6036   src1   : S1(read);
 6037   src2   : S1(read);
 6038   dst    : S3(write);
 6039   INS01  : ISS;
 6040   NEON_FP : S3;
 6041 %}
 6042 
 6043 pipe_class fp_imm_s(vRegF dst)
 6044 %{
 6045   single_instruction;
 6046   dst    : S3(write);
 6047   INS01  : ISS;
 6048   NEON_FP : S3;
 6049 %}
 6050 
 6051 pipe_class fp_imm_d(vRegD dst)
 6052 %{
 6053   single_instruction;
 6054   dst    : S3(write);
 6055   INS01  : ISS;
 6056   NEON_FP : S3;
 6057 %}
 6058 
 6059 pipe_class fp_load_constant_s(vRegF dst)
 6060 %{
 6061   single_instruction;
 6062   dst    : S4(write);
 6063   INS01  : ISS;
 6064   NEON_FP : S4;
 6065 %}
 6066 
 6067 pipe_class fp_load_constant_d(vRegD dst)
 6068 %{
 6069   single_instruction;
 6070   dst    : S4(write);
 6071   INS01  : ISS;
 6072   NEON_FP : S4;
 6073 %}
 6074 
 6075 //------- Integer ALU operations --------------------------
 6076 
 6077 // Integer ALU reg-reg operation
 6078 // Operands needed in EX1, result generated in EX2
 6079 // Eg.  ADD     x0, x1, x2
 6080 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6081 %{
 6082   single_instruction;
 6083   dst    : EX2(write);
 6084   src1   : EX1(read);
 6085   src2   : EX1(read);
 6086   INS01  : ISS; // Dual issue as instruction 0 or 1
 6087   ALU    : EX2;
 6088 %}
 6089 
 6090 // Integer ALU reg-reg operation with constant shift
 6091 // Shifted register must be available in LATE_ISS instead of EX1
 6092 // Eg.  ADD     x0, x1, x2, LSL #2
 6093 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6094 %{
 6095   single_instruction;
 6096   dst    : EX2(write);
 6097   src1   : EX1(read);
 6098   src2   : ISS(read);
 6099   INS01  : ISS;
 6100   ALU    : EX2;
 6101 %}
 6102 
 6103 // Integer ALU reg operation with constant shift
 6104 // Eg.  LSL     x0, x1, #shift
 6105 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6106 %{
 6107   single_instruction;
 6108   dst    : EX2(write);
 6109   src1   : ISS(read);
 6110   INS01  : ISS;
 6111   ALU    : EX2;
 6112 %}
 6113 
 6114 // Integer ALU reg-reg operation with variable shift
 6115 // Both operands must be available in LATE_ISS instead of EX1
 6116 // Result is available in EX1 instead of EX2
 6117 // Eg.  LSLV    x0, x1, x2
 6118 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6119 %{
 6120   single_instruction;
 6121   dst    : EX1(write);
 6122   src1   : ISS(read);
 6123   src2   : ISS(read);
 6124   INS01  : ISS;
 6125   ALU    : EX1;
 6126 %}
 6127 
 6128 // Integer ALU reg-reg operation with extract
 6129 // As for _vshift above, but result generated in EX2
 6130 // Eg.  EXTR    x0, x1, x2, #N
 6131 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6132 %{
 6133   single_instruction;
 6134   dst    : EX2(write);
 6135   src1   : ISS(read);
 6136   src2   : ISS(read);
 6137   INS1   : ISS; // Can only dual issue as Instruction 1
 6138   ALU    : EX1;
 6139 %}
 6140 
 6141 // Integer ALU reg operation
 6142 // Eg.  NEG     x0, x1
 6143 pipe_class ialu_reg(iRegI dst, iRegI src)
 6144 %{
 6145   single_instruction;
 6146   dst    : EX2(write);
 6147   src    : EX1(read);
 6148   INS01  : ISS;
 6149   ALU    : EX2;
 6150 %}
 6151 
 6152 // Integer ALU reg mmediate operation
 6153 // Eg.  ADD     x0, x1, #N
 6154 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6155 %{
 6156   single_instruction;
 6157   dst    : EX2(write);
 6158   src1   : EX1(read);
 6159   INS01  : ISS;
 6160   ALU    : EX2;
 6161 %}
 6162 
 6163 // Integer ALU immediate operation (no source operands)
 6164 // Eg.  MOV     x0, #N
 6165 pipe_class ialu_imm(iRegI dst)
 6166 %{
 6167   single_instruction;
 6168   dst    : EX1(write);
 6169   INS01  : ISS;
 6170   ALU    : EX1;
 6171 %}
 6172 
 6173 //------- Compare operation -------------------------------
 6174 
 6175 // Compare reg-reg
 6176 // Eg.  CMP     x0, x1
 6177 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6178 %{
 6179   single_instruction;
 6180 //  fixed_latency(16);
 6181   cr     : EX2(write);
 6182   op1    : EX1(read);
 6183   op2    : EX1(read);
 6184   INS01  : ISS;
 6185   ALU    : EX2;
 6186 %}
 6187 
 6188 // Compare reg-reg
 6189 // Eg.  CMP     x0, #N
 6190 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6191 %{
 6192   single_instruction;
 6193 //  fixed_latency(16);
 6194   cr     : EX2(write);
 6195   op1    : EX1(read);
 6196   INS01  : ISS;
 6197   ALU    : EX2;
 6198 %}
 6199 
 6200 //------- Conditional instructions ------------------------
 6201 
 6202 // Conditional no operands
 6203 // Eg.  CSINC   x0, zr, zr, <cond>
 6204 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6205 %{
 6206   single_instruction;
 6207   cr     : EX1(read);
 6208   dst    : EX2(write);
 6209   INS01  : ISS;
 6210   ALU    : EX2;
 6211 %}
 6212 
 6213 // Conditional 2 operand
 6214 // EG.  CSEL    X0, X1, X2, <cond>
 6215 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6216 %{
 6217   single_instruction;
 6218   cr     : EX1(read);
 6219   src1   : EX1(read);
 6220   src2   : EX1(read);
 6221   dst    : EX2(write);
 6222   INS01  : ISS;
 6223   ALU    : EX2;
 6224 %}
 6225 
 6226 // Conditional 2 operand
 6227 // EG.  CSEL    X0, X1, X2, <cond>
 6228 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6229 %{
 6230   single_instruction;
 6231   cr     : EX1(read);
 6232   src    : EX1(read);
 6233   dst    : EX2(write);
 6234   INS01  : ISS;
 6235   ALU    : EX2;
 6236 %}
 6237 
 6238 //------- Multiply pipeline operations --------------------
 6239 
 6240 // Multiply reg-reg
 6241 // Eg.  MUL     w0, w1, w2
 6242 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6243 %{
 6244   single_instruction;
 6245   dst    : WR(write);
 6246   src1   : ISS(read);
 6247   src2   : ISS(read);
 6248   INS01  : ISS;
 6249   MAC    : WR;
 6250 %}
 6251 
 6252 // Multiply accumulate
 6253 // Eg.  MADD    w0, w1, w2, w3
 6254 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6255 %{
 6256   single_instruction;
 6257   dst    : WR(write);
 6258   src1   : ISS(read);
 6259   src2   : ISS(read);
 6260   src3   : ISS(read);
 6261   INS01  : ISS;
 6262   MAC    : WR;
 6263 %}
 6264 
 6265 // Eg.  MUL     w0, w1, w2
 6266 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6267 %{
 6268   single_instruction;
 6269   fixed_latency(3); // Maximum latency for 64 bit mul
 6270   dst    : WR(write);
 6271   src1   : ISS(read);
 6272   src2   : ISS(read);
 6273   INS01  : ISS;
 6274   MAC    : WR;
 6275 %}
 6276 
 6277 // Multiply accumulate
 6278 // Eg.  MADD    w0, w1, w2, w3
 6279 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6280 %{
 6281   single_instruction;
 6282   fixed_latency(3); // Maximum latency for 64 bit mul
 6283   dst    : WR(write);
 6284   src1   : ISS(read);
 6285   src2   : ISS(read);
 6286   src3   : ISS(read);
 6287   INS01  : ISS;
 6288   MAC    : WR;
 6289 %}
 6290 
 6291 //------- Divide pipeline operations --------------------
 6292 
 6293 // Eg.  SDIV    w0, w1, w2
 6294 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6295 %{
 6296   single_instruction;
 6297   fixed_latency(8); // Maximum latency for 32 bit divide
 6298   dst    : WR(write);
 6299   src1   : ISS(read);
 6300   src2   : ISS(read);
 6301   INS0   : ISS; // Can only dual issue as instruction 0
 6302   DIV    : WR;
 6303 %}
 6304 
 6305 // Eg.  SDIV    x0, x1, x2
 6306 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6307 %{
 6308   single_instruction;
 6309   fixed_latency(16); // Maximum latency for 64 bit divide
 6310   dst    : WR(write);
 6311   src1   : ISS(read);
 6312   src2   : ISS(read);
 6313   INS0   : ISS; // Can only dual issue as instruction 0
 6314   DIV    : WR;
 6315 %}
 6316 
 6317 //------- Load pipeline operations ------------------------
 6318 
 6319 // Load - prefetch
 6320 // Eg.  PFRM    <mem>
 6321 pipe_class iload_prefetch(memory mem)
 6322 %{
 6323   single_instruction;
 6324   mem    : ISS(read);
 6325   INS01  : ISS;
 6326   LDST   : WR;
 6327 %}
 6328 
 6329 // Load - reg, mem
 6330 // Eg.  LDR     x0, <mem>
 6331 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6332 %{
 6333   single_instruction;
 6334   dst    : WR(write);
 6335   mem    : ISS(read);
 6336   INS01  : ISS;
 6337   LDST   : WR;
 6338 %}
 6339 
 6340 // Load - reg, reg
 6341 // Eg.  LDR     x0, [sp, x1]
 6342 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6343 %{
 6344   single_instruction;
 6345   dst    : WR(write);
 6346   src    : ISS(read);
 6347   INS01  : ISS;
 6348   LDST   : WR;
 6349 %}
 6350 
 6351 //------- Store pipeline operations -----------------------
 6352 
 6353 // Store - zr, mem
 6354 // Eg.  STR     zr, <mem>
 6355 pipe_class istore_mem(memory mem)
 6356 %{
 6357   single_instruction;
 6358   mem    : ISS(read);
 6359   INS01  : ISS;
 6360   LDST   : WR;
 6361 %}
 6362 
 6363 // Store - reg, mem
 6364 // Eg.  STR     x0, <mem>
 6365 pipe_class istore_reg_mem(iRegI src, memory mem)
 6366 %{
 6367   single_instruction;
 6368   mem    : ISS(read);
 6369   src    : EX2(read);
 6370   INS01  : ISS;
 6371   LDST   : WR;
 6372 %}
 6373 
 6374 // Store - reg, reg
 6375 // Eg. STR      x0, [sp, x1]
 6376 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6377 %{
 6378   single_instruction;
 6379   dst    : ISS(read);
 6380   src    : EX2(read);
 6381   INS01  : ISS;
 6382   LDST   : WR;
 6383 %}
 6384 
 6385 //------- Store pipeline operations -----------------------
 6386 
 6387 // Branch
 6388 pipe_class pipe_branch()
 6389 %{
 6390   single_instruction;
 6391   INS01  : ISS;
 6392   BRANCH : EX1;
 6393 %}
 6394 
 6395 // Conditional branch
 6396 pipe_class pipe_branch_cond(rFlagsReg cr)
 6397 %{
 6398   single_instruction;
 6399   cr     : EX1(read);
 6400   INS01  : ISS;
 6401   BRANCH : EX1;
 6402 %}
 6403 
 6404 // Compare & Branch
 6405 // EG.  CBZ/CBNZ
 6406 pipe_class pipe_cmp_branch(iRegI op1)
 6407 %{
 6408   single_instruction;
 6409   op1    : EX1(read);
 6410   INS01  : ISS;
 6411   BRANCH : EX1;
 6412 %}
 6413 
 6414 //------- Synchronisation operations ----------------------
 6415 
 6416 // Any operation requiring serialization.
 6417 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6418 pipe_class pipe_serial()
 6419 %{
 6420   single_instruction;
 6421   force_serialization;
 6422   fixed_latency(16);
 6423   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6424   LDST   : WR;
 6425 %}
 6426 
 6427 // Generic big/slow expanded idiom - also serialized
 6428 pipe_class pipe_slow()
 6429 %{
 6430   instruction_count(10);
 6431   multiple_bundles;
 6432   force_serialization;
 6433   fixed_latency(16);
 6434   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6435   LDST   : WR;
 6436 %}
 6437 
 6438 // Empty pipeline class
 6439 pipe_class pipe_class_empty()
 6440 %{
 6441   single_instruction;
 6442   fixed_latency(0);
 6443 %}
 6444 
 6445 // Default pipeline class.
 6446 pipe_class pipe_class_default()
 6447 %{
 6448   single_instruction;
 6449   fixed_latency(2);
 6450 %}
 6451 
 6452 // Pipeline class for compares.
 6453 pipe_class pipe_class_compare()
 6454 %{
 6455   single_instruction;
 6456   fixed_latency(16);
 6457 %}
 6458 
 6459 // Pipeline class for memory operations.
 6460 pipe_class pipe_class_memory()
 6461 %{
 6462   single_instruction;
 6463   fixed_latency(16);
 6464 %}
 6465 
 6466 // Pipeline class for call.
 6467 pipe_class pipe_class_call()
 6468 %{
 6469   single_instruction;
 6470   fixed_latency(100);
 6471 %}
 6472 
 6473 // Define the class for the Nop node.
 6474 define %{
 6475    MachNop = pipe_class_empty;
 6476 %}
 6477 
 6478 %}
 6479 //----------INSTRUCTIONS-------------------------------------------------------
 6480 //
 6481 // match      -- States which machine-independent subtree may be replaced
 6482 //               by this instruction.
 6483 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6484 //               selection to identify a minimum cost tree of machine
 6485 //               instructions that matches a tree of machine-independent
 6486 //               instructions.
 6487 // format     -- A string providing the disassembly for this instruction.
 6488 //               The value of an instruction's operand may be inserted
 6489 //               by referring to it with a '$' prefix.
 6490 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6491 //               to within an encode class as $primary, $secondary, and $tertiary
 6492 //               rrspectively.  The primary opcode is commonly used to
 6493 //               indicate the type of machine instruction, while secondary
 6494 //               and tertiary are often used for prefix options or addressing
 6495 //               modes.
 6496 // ins_encode -- A list of encode classes with parameters. The encode class
 6497 //               name must have been defined in an 'enc_class' specification
 6498 //               in the encode section of the architecture description.
 6499 
 6500 // ============================================================================
 6501 // Memory (Load/Store) Instructions
 6502 
 6503 // Load Instructions
 6504 
 6505 // Load Byte (8 bit signed)
 6506 instruct loadB(iRegINoSp dst, memory1 mem)
 6507 %{
 6508   match(Set dst (LoadB mem));
 6509   predicate(!needs_acquiring_load(n));
 6510 
 6511   ins_cost(4 * INSN_COST);
 6512   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6513 
 6514   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6515 
 6516   ins_pipe(iload_reg_mem);
 6517 %}
 6518 
 6519 // Load Byte (8 bit signed) into long
 6520 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6521 %{
 6522   match(Set dst (ConvI2L (LoadB mem)));
 6523   predicate(!needs_acquiring_load(n->in(1)));
 6524 
 6525   ins_cost(4 * INSN_COST);
 6526   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6527 
 6528   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6529 
 6530   ins_pipe(iload_reg_mem);
 6531 %}
 6532 
 6533 // Load Byte (8 bit unsigned)
 6534 instruct loadUB(iRegINoSp dst, memory1 mem)
 6535 %{
 6536   match(Set dst (LoadUB mem));
 6537   predicate(!needs_acquiring_load(n));
 6538 
 6539   ins_cost(4 * INSN_COST);
 6540   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6541 
 6542   ins_encode(aarch64_enc_ldrb(dst, mem));
 6543 
 6544   ins_pipe(iload_reg_mem);
 6545 %}
 6546 
 6547 // Load Byte (8 bit unsigned) into long
 6548 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6549 %{
 6550   match(Set dst (ConvI2L (LoadUB mem)));
 6551   predicate(!needs_acquiring_load(n->in(1)));
 6552 
 6553   ins_cost(4 * INSN_COST);
 6554   format %{ "ldrb  $dst, $mem\t# byte" %}
 6555 
 6556   ins_encode(aarch64_enc_ldrb(dst, mem));
 6557 
 6558   ins_pipe(iload_reg_mem);
 6559 %}
 6560 
 6561 // Load Short (16 bit signed)
 6562 instruct loadS(iRegINoSp dst, memory2 mem)
 6563 %{
 6564   match(Set dst (LoadS mem));
 6565   predicate(!needs_acquiring_load(n));
 6566 
 6567   ins_cost(4 * INSN_COST);
 6568   format %{ "ldrshw  $dst, $mem\t# short" %}
 6569 
 6570   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6571 
 6572   ins_pipe(iload_reg_mem);
 6573 %}
 6574 
 6575 // Load Short (16 bit signed) into long
 6576 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6577 %{
 6578   match(Set dst (ConvI2L (LoadS mem)));
 6579   predicate(!needs_acquiring_load(n->in(1)));
 6580 
 6581   ins_cost(4 * INSN_COST);
 6582   format %{ "ldrsh  $dst, $mem\t# short" %}
 6583 
 6584   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6585 
 6586   ins_pipe(iload_reg_mem);
 6587 %}
 6588 
 6589 // Load Char (16 bit unsigned)
 6590 instruct loadUS(iRegINoSp dst, memory2 mem)
 6591 %{
 6592   match(Set dst (LoadUS mem));
 6593   predicate(!needs_acquiring_load(n));
 6594 
 6595   ins_cost(4 * INSN_COST);
 6596   format %{ "ldrh  $dst, $mem\t# short" %}
 6597 
 6598   ins_encode(aarch64_enc_ldrh(dst, mem));
 6599 
 6600   ins_pipe(iload_reg_mem);
 6601 %}
 6602 
 6603 // Load Short/Char (16 bit unsigned) into long
 6604 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6605 %{
 6606   match(Set dst (ConvI2L (LoadUS mem)));
 6607   predicate(!needs_acquiring_load(n->in(1)));
 6608 
 6609   ins_cost(4 * INSN_COST);
 6610   format %{ "ldrh  $dst, $mem\t# short" %}
 6611 
 6612   ins_encode(aarch64_enc_ldrh(dst, mem));
 6613 
 6614   ins_pipe(iload_reg_mem);
 6615 %}
 6616 
 6617 // Load Integer (32 bit signed)
 6618 instruct loadI(iRegINoSp dst, memory4 mem)
 6619 %{
 6620   match(Set dst (LoadI mem));
 6621   predicate(!needs_acquiring_load(n));
 6622 
 6623   ins_cost(4 * INSN_COST);
 6624   format %{ "ldrw  $dst, $mem\t# int" %}
 6625 
 6626   ins_encode(aarch64_enc_ldrw(dst, mem));
 6627 
 6628   ins_pipe(iload_reg_mem);
 6629 %}
 6630 
 6631 // Load Integer (32 bit signed) into long
 6632 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6633 %{
 6634   match(Set dst (ConvI2L (LoadI mem)));
 6635   predicate(!needs_acquiring_load(n->in(1)));
 6636 
 6637   ins_cost(4 * INSN_COST);
 6638   format %{ "ldrsw  $dst, $mem\t# int" %}
 6639 
 6640   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6641 
 6642   ins_pipe(iload_reg_mem);
 6643 %}
 6644 
 6645 // Load Integer (32 bit unsigned) into long
 6646 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6647 %{
 6648   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6649   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6650 
 6651   ins_cost(4 * INSN_COST);
 6652   format %{ "ldrw  $dst, $mem\t# int" %}
 6653 
 6654   ins_encode(aarch64_enc_ldrw(dst, mem));
 6655 
 6656   ins_pipe(iload_reg_mem);
 6657 %}
 6658 
 6659 // Load Long (64 bit signed)
 6660 instruct loadL(iRegLNoSp dst, memory8 mem)
 6661 %{
 6662   match(Set dst (LoadL mem));
 6663   predicate(!needs_acquiring_load(n));
 6664 
 6665   ins_cost(4 * INSN_COST);
 6666   format %{ "ldr  $dst, $mem\t# int" %}
 6667 
 6668   ins_encode(aarch64_enc_ldr(dst, mem));
 6669 
 6670   ins_pipe(iload_reg_mem);
 6671 %}
 6672 
 6673 // Load Range
 6674 instruct loadRange(iRegINoSp dst, memory4 mem)
 6675 %{
 6676   match(Set dst (LoadRange mem));
 6677 
 6678   ins_cost(4 * INSN_COST);
 6679   format %{ "ldrw  $dst, $mem\t# range" %}
 6680 
 6681   ins_encode(aarch64_enc_ldrw(dst, mem));
 6682 
 6683   ins_pipe(iload_reg_mem);
 6684 %}
 6685 
 6686 // Load Pointer
 6687 instruct loadP(iRegPNoSp dst, memory8 mem)
 6688 %{
 6689   match(Set dst (LoadP mem));
 6690   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6691 
 6692   ins_cost(4 * INSN_COST);
 6693   format %{ "ldr  $dst, $mem\t# ptr" %}
 6694 
 6695   ins_encode(aarch64_enc_ldr(dst, mem));
 6696 
 6697   ins_pipe(iload_reg_mem);
 6698 %}
 6699 
 6700 // Load Compressed Pointer
 6701 instruct loadN(iRegNNoSp dst, memory4 mem)
 6702 %{
 6703   match(Set dst (LoadN mem));
 6704   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6705 
 6706   ins_cost(4 * INSN_COST);
 6707   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6708 
 6709   ins_encode(aarch64_enc_ldrw(dst, mem));
 6710 
 6711   ins_pipe(iload_reg_mem);
 6712 %}
 6713 
 6714 // Load Klass Pointer
 6715 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6716 %{
 6717   match(Set dst (LoadKlass mem));
 6718   predicate(!needs_acquiring_load(n));
 6719 
 6720   ins_cost(4 * INSN_COST);
 6721   format %{ "ldr  $dst, $mem\t# class" %}
 6722 
 6723   ins_encode(aarch64_enc_ldr(dst, mem));
 6724 
 6725   ins_pipe(iload_reg_mem);
 6726 %}
 6727 
 6728 // Load Narrow Klass Pointer
 6729 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6730 %{
 6731   match(Set dst (LoadNKlass mem));
 6732   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6733 
 6734   ins_cost(4 * INSN_COST);
 6735   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6736 
 6737   ins_encode(aarch64_enc_ldrw(dst, mem));
 6738 
 6739   ins_pipe(iload_reg_mem);
 6740 %}
 6741 
 6742 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
 6743 %{
 6744   match(Set dst (LoadNKlass mem));
 6745   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6746 
 6747   ins_cost(4 * INSN_COST);
 6748   format %{
 6749     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6750     "lsrw  $dst, $dst, markWord::klass_shift"
 6751   %}
 6752   ins_encode %{
 6753     assert($mem$$index$$Register == noreg, "must not have indexed address");
 6754     // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
 6755     // obj-start, so that we can load from the object's mark-word instead.
 6756     __ ldrw($dst$$Register, Address($mem$$base$$Register, $mem$$disp - Type::klass_offset()));
 6757     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift);
 6758   %}
 6759   ins_pipe(iload_reg_mem);
 6760 %}
 6761 
 6762 // Load Float
 6763 instruct loadF(vRegF dst, memory4 mem)
 6764 %{
 6765   match(Set dst (LoadF mem));
 6766   predicate(!needs_acquiring_load(n));
 6767 
 6768   ins_cost(4 * INSN_COST);
 6769   format %{ "ldrs  $dst, $mem\t# float" %}
 6770 
 6771   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6772 
 6773   ins_pipe(pipe_class_memory);
 6774 %}
 6775 
 6776 // Load Double
 6777 instruct loadD(vRegD dst, memory8 mem)
 6778 %{
 6779   match(Set dst (LoadD mem));
 6780   predicate(!needs_acquiring_load(n));
 6781 
 6782   ins_cost(4 * INSN_COST);
 6783   format %{ "ldrd  $dst, $mem\t# double" %}
 6784 
 6785   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6786 
 6787   ins_pipe(pipe_class_memory);
 6788 %}
 6789 
 6790 
 6791 // Load Int Constant
 6792 instruct loadConI(iRegINoSp dst, immI src)
 6793 %{
 6794   match(Set dst src);
 6795 
 6796   ins_cost(INSN_COST);
 6797   format %{ "mov $dst, $src\t# int" %}
 6798 
 6799   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6800 
 6801   ins_pipe(ialu_imm);
 6802 %}
 6803 
 6804 // Load Long Constant
 6805 instruct loadConL(iRegLNoSp dst, immL src)
 6806 %{
 6807   match(Set dst src);
 6808 
 6809   ins_cost(INSN_COST);
 6810   format %{ "mov $dst, $src\t# long" %}
 6811 
 6812   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6813 
 6814   ins_pipe(ialu_imm);
 6815 %}
 6816 
 6817 // Load Pointer Constant
 6818 
 6819 instruct loadConP(iRegPNoSp dst, immP con)
 6820 %{
 6821   match(Set dst con);
 6822 
 6823   ins_cost(INSN_COST * 4);
 6824   format %{
 6825     "mov  $dst, $con\t# ptr\n\t"
 6826   %}
 6827 
 6828   ins_encode(aarch64_enc_mov_p(dst, con));
 6829 
 6830   ins_pipe(ialu_imm);
 6831 %}
 6832 
 6833 // Load Null Pointer Constant
 6834 
 6835 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6836 %{
 6837   match(Set dst con);
 6838 
 6839   ins_cost(INSN_COST);
 6840   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6841 
 6842   ins_encode(aarch64_enc_mov_p0(dst, con));
 6843 
 6844   ins_pipe(ialu_imm);
 6845 %}
 6846 
 6847 // Load Pointer Constant One
 6848 
 6849 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6850 %{
 6851   match(Set dst con);
 6852 
 6853   ins_cost(INSN_COST);
 6854   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6855 
 6856   ins_encode(aarch64_enc_mov_p1(dst, con));
 6857 
 6858   ins_pipe(ialu_imm);
 6859 %}
 6860 
 6861 // Load Byte Map Base Constant
 6862 
 6863 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 6864 %{
 6865   match(Set dst con);
 6866 
 6867   ins_cost(INSN_COST);
 6868   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 6869 
 6870   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 6871 
 6872   ins_pipe(ialu_imm);
 6873 %}
 6874 
 6875 // Load Narrow Pointer Constant
 6876 
 6877 instruct loadConN(iRegNNoSp dst, immN con)
 6878 %{
 6879   match(Set dst con);
 6880 
 6881   ins_cost(INSN_COST * 4);
 6882   format %{ "mov  $dst, $con\t# compressed ptr" %}
 6883 
 6884   ins_encode(aarch64_enc_mov_n(dst, con));
 6885 
 6886   ins_pipe(ialu_imm);
 6887 %}
 6888 
 6889 // Load Narrow Null Pointer Constant
 6890 
 6891 instruct loadConN0(iRegNNoSp dst, immN0 con)
 6892 %{
 6893   match(Set dst con);
 6894 
 6895   ins_cost(INSN_COST);
 6896   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 6897 
 6898   ins_encode(aarch64_enc_mov_n0(dst, con));
 6899 
 6900   ins_pipe(ialu_imm);
 6901 %}
 6902 
 6903 // Load Narrow Klass Constant
 6904 
 6905 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 6906 %{
 6907   match(Set dst con);
 6908 
 6909   ins_cost(INSN_COST);
 6910   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 6911 
 6912   ins_encode(aarch64_enc_mov_nk(dst, con));
 6913 
 6914   ins_pipe(ialu_imm);
 6915 %}
 6916 
 6917 // Load Packed Float Constant
 6918 
 6919 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 6920   match(Set dst con);
 6921   ins_cost(INSN_COST * 4);
 6922   format %{ "fmovs  $dst, $con"%}
 6923   ins_encode %{
 6924     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 6925   %}
 6926 
 6927   ins_pipe(fp_imm_s);
 6928 %}
 6929 
 6930 // Load Float Constant
 6931 
 6932 instruct loadConF(vRegF dst, immF con) %{
 6933   match(Set dst con);
 6934 
 6935   ins_cost(INSN_COST * 4);
 6936 
 6937   format %{
 6938     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6939   %}
 6940 
 6941   ins_encode %{
 6942     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 6943   %}
 6944 
 6945   ins_pipe(fp_load_constant_s);
 6946 %}
 6947 
 6948 // Load Packed Double Constant
 6949 
 6950 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 6951   match(Set dst con);
 6952   ins_cost(INSN_COST);
 6953   format %{ "fmovd  $dst, $con"%}
 6954   ins_encode %{
 6955     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 6956   %}
 6957 
 6958   ins_pipe(fp_imm_d);
 6959 %}
 6960 
 6961 // Load Double Constant
 6962 
 6963 instruct loadConD(vRegD dst, immD con) %{
 6964   match(Set dst con);
 6965 
 6966   ins_cost(INSN_COST * 5);
 6967   format %{
 6968     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6969   %}
 6970 
 6971   ins_encode %{
 6972     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 6973   %}
 6974 
 6975   ins_pipe(fp_load_constant_d);
 6976 %}
 6977 
 6978 // Load Half Float Constant
 6979 // The "ldr" instruction loads a 32-bit word from the constant pool into a
 6980 // 32-bit register but only the bottom half will be populated and the top
 6981 // 16 bits are zero.
 6982 instruct loadConH(vRegF dst, immH con) %{
 6983   match(Set dst con);
 6984   format %{
 6985     "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
 6986   %}
 6987   ins_encode %{
 6988     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 6989   %}
 6990   ins_pipe(fp_load_constant_s);
 6991 %}
 6992 
 6993 // Store Instructions
 6994 
 6995 // Store Byte
 6996 instruct storeB(iRegIorL2I src, memory1 mem)
 6997 %{
 6998   match(Set mem (StoreB mem src));
 6999   predicate(!needs_releasing_store(n));
 7000 
 7001   ins_cost(INSN_COST);
 7002   format %{ "strb  $src, $mem\t# byte" %}
 7003 
 7004   ins_encode(aarch64_enc_strb(src, mem));
 7005 
 7006   ins_pipe(istore_reg_mem);
 7007 %}
 7008 
 7009 
 7010 instruct storeimmB0(immI0 zero, memory1 mem)
 7011 %{
 7012   match(Set mem (StoreB mem zero));
 7013   predicate(!needs_releasing_store(n));
 7014 
 7015   ins_cost(INSN_COST);
 7016   format %{ "strb rscractch2, $mem\t# byte" %}
 7017 
 7018   ins_encode(aarch64_enc_strb0(mem));
 7019 
 7020   ins_pipe(istore_mem);
 7021 %}
 7022 
 7023 // Store Char/Short
 7024 instruct storeC(iRegIorL2I src, memory2 mem)
 7025 %{
 7026   match(Set mem (StoreC mem src));
 7027   predicate(!needs_releasing_store(n));
 7028 
 7029   ins_cost(INSN_COST);
 7030   format %{ "strh  $src, $mem\t# short" %}
 7031 
 7032   ins_encode(aarch64_enc_strh(src, mem));
 7033 
 7034   ins_pipe(istore_reg_mem);
 7035 %}
 7036 
 7037 instruct storeimmC0(immI0 zero, memory2 mem)
 7038 %{
 7039   match(Set mem (StoreC mem zero));
 7040   predicate(!needs_releasing_store(n));
 7041 
 7042   ins_cost(INSN_COST);
 7043   format %{ "strh  zr, $mem\t# short" %}
 7044 
 7045   ins_encode(aarch64_enc_strh0(mem));
 7046 
 7047   ins_pipe(istore_mem);
 7048 %}
 7049 
 7050 // Store Integer
 7051 
 7052 instruct storeI(iRegIorL2I src, memory4 mem)
 7053 %{
 7054   match(Set mem(StoreI mem src));
 7055   predicate(!needs_releasing_store(n));
 7056 
 7057   ins_cost(INSN_COST);
 7058   format %{ "strw  $src, $mem\t# int" %}
 7059 
 7060   ins_encode(aarch64_enc_strw(src, mem));
 7061 
 7062   ins_pipe(istore_reg_mem);
 7063 %}
 7064 
 7065 instruct storeimmI0(immI0 zero, memory4 mem)
 7066 %{
 7067   match(Set mem(StoreI mem zero));
 7068   predicate(!needs_releasing_store(n));
 7069 
 7070   ins_cost(INSN_COST);
 7071   format %{ "strw  zr, $mem\t# int" %}
 7072 
 7073   ins_encode(aarch64_enc_strw0(mem));
 7074 
 7075   ins_pipe(istore_mem);
 7076 %}
 7077 
 7078 // Store Long (64 bit signed)
 7079 instruct storeL(iRegL src, memory8 mem)
 7080 %{
 7081   match(Set mem (StoreL mem src));
 7082   predicate(!needs_releasing_store(n));
 7083 
 7084   ins_cost(INSN_COST);
 7085   format %{ "str  $src, $mem\t# int" %}
 7086 
 7087   ins_encode(aarch64_enc_str(src, mem));
 7088 
 7089   ins_pipe(istore_reg_mem);
 7090 %}
 7091 
 7092 // Store Long (64 bit signed)
 7093 instruct storeimmL0(immL0 zero, memory8 mem)
 7094 %{
 7095   match(Set mem (StoreL mem zero));
 7096   predicate(!needs_releasing_store(n));
 7097 
 7098   ins_cost(INSN_COST);
 7099   format %{ "str  zr, $mem\t# int" %}
 7100 
 7101   ins_encode(aarch64_enc_str0(mem));
 7102 
 7103   ins_pipe(istore_mem);
 7104 %}
 7105 
 7106 // Store Pointer
 7107 instruct storeP(iRegP src, memory8 mem)
 7108 %{
 7109   match(Set mem (StoreP mem src));
 7110   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7111 
 7112   ins_cost(INSN_COST);
 7113   format %{ "str  $src, $mem\t# ptr" %}
 7114 
 7115   ins_encode(aarch64_enc_str(src, mem));
 7116 
 7117   ins_pipe(istore_reg_mem);
 7118 %}
 7119 
 7120 // Store Pointer
 7121 instruct storeimmP0(immP0 zero, memory8 mem)
 7122 %{
 7123   match(Set mem (StoreP mem zero));
 7124   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7125 
 7126   ins_cost(INSN_COST);
 7127   format %{ "str zr, $mem\t# ptr" %}
 7128 
 7129   ins_encode(aarch64_enc_str0(mem));
 7130 
 7131   ins_pipe(istore_mem);
 7132 %}
 7133 
 7134 // Store Compressed Pointer
 7135 instruct storeN(iRegN src, memory4 mem)
 7136 %{
 7137   match(Set mem (StoreN mem src));
 7138   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7139 
 7140   ins_cost(INSN_COST);
 7141   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7142 
 7143   ins_encode(aarch64_enc_strw(src, mem));
 7144 
 7145   ins_pipe(istore_reg_mem);
 7146 %}
 7147 
 7148 instruct storeImmN0(immN0 zero, memory4 mem)
 7149 %{
 7150   match(Set mem (StoreN mem zero));
 7151   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7152 
 7153   ins_cost(INSN_COST);
 7154   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7155 
 7156   ins_encode(aarch64_enc_strw0(mem));
 7157 
 7158   ins_pipe(istore_mem);
 7159 %}
 7160 
 7161 // Store Float
 7162 instruct storeF(vRegF src, memory4 mem)
 7163 %{
 7164   match(Set mem (StoreF mem src));
 7165   predicate(!needs_releasing_store(n));
 7166 
 7167   ins_cost(INSN_COST);
 7168   format %{ "strs  $src, $mem\t# float" %}
 7169 
 7170   ins_encode( aarch64_enc_strs(src, mem) );
 7171 
 7172   ins_pipe(pipe_class_memory);
 7173 %}
 7174 
 7175 // TODO
 7176 // implement storeImmF0 and storeFImmPacked
 7177 
 7178 // Store Double
 7179 instruct storeD(vRegD src, memory8 mem)
 7180 %{
 7181   match(Set mem (StoreD mem src));
 7182   predicate(!needs_releasing_store(n));
 7183 
 7184   ins_cost(INSN_COST);
 7185   format %{ "strd  $src, $mem\t# double" %}
 7186 
 7187   ins_encode( aarch64_enc_strd(src, mem) );
 7188 
 7189   ins_pipe(pipe_class_memory);
 7190 %}
 7191 
 7192 // Store Compressed Klass Pointer
 7193 instruct storeNKlass(iRegN src, memory4 mem)
 7194 %{
 7195   predicate(!needs_releasing_store(n));
 7196   match(Set mem (StoreNKlass mem src));
 7197 
 7198   ins_cost(INSN_COST);
 7199   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7200 
 7201   ins_encode(aarch64_enc_strw(src, mem));
 7202 
 7203   ins_pipe(istore_reg_mem);
 7204 %}
 7205 
 7206 // TODO
 7207 // implement storeImmD0 and storeDImmPacked
 7208 
 7209 // prefetch instructions
 7210 // Must be safe to execute with invalid address (cannot fault).
 7211 
 7212 instruct prefetchalloc( memory8 mem ) %{
 7213   match(PrefetchAllocation mem);
 7214 
 7215   ins_cost(INSN_COST);
 7216   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7217 
 7218   ins_encode( aarch64_enc_prefetchw(mem) );
 7219 
 7220   ins_pipe(iload_prefetch);
 7221 %}
 7222 
 7223 //  ---------------- volatile loads and stores ----------------
 7224 
 7225 // Load Byte (8 bit signed)
 7226 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7227 %{
 7228   match(Set dst (LoadB mem));
 7229 
 7230   ins_cost(VOLATILE_REF_COST);
 7231   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7232 
 7233   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7234 
 7235   ins_pipe(pipe_serial);
 7236 %}
 7237 
 7238 // Load Byte (8 bit signed) into long
 7239 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7240 %{
 7241   match(Set dst (ConvI2L (LoadB mem)));
 7242 
 7243   ins_cost(VOLATILE_REF_COST);
 7244   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7245 
 7246   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7247 
 7248   ins_pipe(pipe_serial);
 7249 %}
 7250 
 7251 // Load Byte (8 bit unsigned)
 7252 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7253 %{
 7254   match(Set dst (LoadUB mem));
 7255 
 7256   ins_cost(VOLATILE_REF_COST);
 7257   format %{ "ldarb  $dst, $mem\t# byte" %}
 7258 
 7259   ins_encode(aarch64_enc_ldarb(dst, mem));
 7260 
 7261   ins_pipe(pipe_serial);
 7262 %}
 7263 
 7264 // Load Byte (8 bit unsigned) into long
 7265 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7266 %{
 7267   match(Set dst (ConvI2L (LoadUB mem)));
 7268 
 7269   ins_cost(VOLATILE_REF_COST);
 7270   format %{ "ldarb  $dst, $mem\t# byte" %}
 7271 
 7272   ins_encode(aarch64_enc_ldarb(dst, mem));
 7273 
 7274   ins_pipe(pipe_serial);
 7275 %}
 7276 
 7277 // Load Short (16 bit signed)
 7278 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7279 %{
 7280   match(Set dst (LoadS mem));
 7281 
 7282   ins_cost(VOLATILE_REF_COST);
 7283   format %{ "ldarshw  $dst, $mem\t# short" %}
 7284 
 7285   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7286 
 7287   ins_pipe(pipe_serial);
 7288 %}
 7289 
 7290 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7291 %{
 7292   match(Set dst (LoadUS mem));
 7293 
 7294   ins_cost(VOLATILE_REF_COST);
 7295   format %{ "ldarhw  $dst, $mem\t# short" %}
 7296 
 7297   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7298 
 7299   ins_pipe(pipe_serial);
 7300 %}
 7301 
 7302 // Load Short/Char (16 bit unsigned) into long
 7303 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7304 %{
 7305   match(Set dst (ConvI2L (LoadUS mem)));
 7306 
 7307   ins_cost(VOLATILE_REF_COST);
 7308   format %{ "ldarh  $dst, $mem\t# short" %}
 7309 
 7310   ins_encode(aarch64_enc_ldarh(dst, mem));
 7311 
 7312   ins_pipe(pipe_serial);
 7313 %}
 7314 
 7315 // Load Short/Char (16 bit signed) into long
 7316 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7317 %{
 7318   match(Set dst (ConvI2L (LoadS mem)));
 7319 
 7320   ins_cost(VOLATILE_REF_COST);
 7321   format %{ "ldarh  $dst, $mem\t# short" %}
 7322 
 7323   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7324 
 7325   ins_pipe(pipe_serial);
 7326 %}
 7327 
 7328 // Load Integer (32 bit signed)
 7329 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7330 %{
 7331   match(Set dst (LoadI mem));
 7332 
 7333   ins_cost(VOLATILE_REF_COST);
 7334   format %{ "ldarw  $dst, $mem\t# int" %}
 7335 
 7336   ins_encode(aarch64_enc_ldarw(dst, mem));
 7337 
 7338   ins_pipe(pipe_serial);
 7339 %}
 7340 
 7341 // Load Integer (32 bit unsigned) into long
 7342 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7343 %{
 7344   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7345 
 7346   ins_cost(VOLATILE_REF_COST);
 7347   format %{ "ldarw  $dst, $mem\t# int" %}
 7348 
 7349   ins_encode(aarch64_enc_ldarw(dst, mem));
 7350 
 7351   ins_pipe(pipe_serial);
 7352 %}
 7353 
 7354 // Load Long (64 bit signed)
 7355 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7356 %{
 7357   match(Set dst (LoadL mem));
 7358 
 7359   ins_cost(VOLATILE_REF_COST);
 7360   format %{ "ldar  $dst, $mem\t# int" %}
 7361 
 7362   ins_encode(aarch64_enc_ldar(dst, mem));
 7363 
 7364   ins_pipe(pipe_serial);
 7365 %}
 7366 
 7367 // Load Pointer
 7368 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7369 %{
 7370   match(Set dst (LoadP mem));
 7371   predicate(n->as_Load()->barrier_data() == 0);
 7372 
 7373   ins_cost(VOLATILE_REF_COST);
 7374   format %{ "ldar  $dst, $mem\t# ptr" %}
 7375 
 7376   ins_encode(aarch64_enc_ldar(dst, mem));
 7377 
 7378   ins_pipe(pipe_serial);
 7379 %}
 7380 
 7381 // Load Compressed Pointer
 7382 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7383 %{
 7384   match(Set dst (LoadN mem));
 7385   predicate(n->as_Load()->barrier_data() == 0);
 7386 
 7387   ins_cost(VOLATILE_REF_COST);
 7388   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7389 
 7390   ins_encode(aarch64_enc_ldarw(dst, mem));
 7391 
 7392   ins_pipe(pipe_serial);
 7393 %}
 7394 
 7395 // Load Float
 7396 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7397 %{
 7398   match(Set dst (LoadF mem));
 7399 
 7400   ins_cost(VOLATILE_REF_COST);
 7401   format %{ "ldars  $dst, $mem\t# float" %}
 7402 
 7403   ins_encode( aarch64_enc_fldars(dst, mem) );
 7404 
 7405   ins_pipe(pipe_serial);
 7406 %}
 7407 
 7408 // Load Double
 7409 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7410 %{
 7411   match(Set dst (LoadD mem));
 7412 
 7413   ins_cost(VOLATILE_REF_COST);
 7414   format %{ "ldard  $dst, $mem\t# double" %}
 7415 
 7416   ins_encode( aarch64_enc_fldard(dst, mem) );
 7417 
 7418   ins_pipe(pipe_serial);
 7419 %}
 7420 
 7421 // Store Byte
 7422 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7423 %{
 7424   match(Set mem (StoreB mem src));
 7425 
 7426   ins_cost(VOLATILE_REF_COST);
 7427   format %{ "stlrb  $src, $mem\t# byte" %}
 7428 
 7429   ins_encode(aarch64_enc_stlrb(src, mem));
 7430 
 7431   ins_pipe(pipe_class_memory);
 7432 %}
 7433 
 7434 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7435 %{
 7436   match(Set mem (StoreB mem zero));
 7437 
 7438   ins_cost(VOLATILE_REF_COST);
 7439   format %{ "stlrb  zr, $mem\t# byte" %}
 7440 
 7441   ins_encode(aarch64_enc_stlrb0(mem));
 7442 
 7443   ins_pipe(pipe_class_memory);
 7444 %}
 7445 
 7446 // Store Char/Short
 7447 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7448 %{
 7449   match(Set mem (StoreC mem src));
 7450 
 7451   ins_cost(VOLATILE_REF_COST);
 7452   format %{ "stlrh  $src, $mem\t# short" %}
 7453 
 7454   ins_encode(aarch64_enc_stlrh(src, mem));
 7455 
 7456   ins_pipe(pipe_class_memory);
 7457 %}
 7458 
 7459 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7460 %{
 7461   match(Set mem (StoreC mem zero));
 7462 
 7463   ins_cost(VOLATILE_REF_COST);
 7464   format %{ "stlrh  zr, $mem\t# short" %}
 7465 
 7466   ins_encode(aarch64_enc_stlrh0(mem));
 7467 
 7468   ins_pipe(pipe_class_memory);
 7469 %}
 7470 
 7471 // Store Integer
 7472 
 7473 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7474 %{
 7475   match(Set mem(StoreI mem src));
 7476 
 7477   ins_cost(VOLATILE_REF_COST);
 7478   format %{ "stlrw  $src, $mem\t# int" %}
 7479 
 7480   ins_encode(aarch64_enc_stlrw(src, mem));
 7481 
 7482   ins_pipe(pipe_class_memory);
 7483 %}
 7484 
 7485 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7486 %{
 7487   match(Set mem(StoreI mem zero));
 7488 
 7489   ins_cost(VOLATILE_REF_COST);
 7490   format %{ "stlrw  zr, $mem\t# int" %}
 7491 
 7492   ins_encode(aarch64_enc_stlrw0(mem));
 7493 
 7494   ins_pipe(pipe_class_memory);
 7495 %}
 7496 
 7497 // Store Long (64 bit signed)
 7498 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7499 %{
 7500   match(Set mem (StoreL mem src));
 7501 
 7502   ins_cost(VOLATILE_REF_COST);
 7503   format %{ "stlr  $src, $mem\t# int" %}
 7504 
 7505   ins_encode(aarch64_enc_stlr(src, mem));
 7506 
 7507   ins_pipe(pipe_class_memory);
 7508 %}
 7509 
 7510 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7511 %{
 7512   match(Set mem (StoreL mem zero));
 7513 
 7514   ins_cost(VOLATILE_REF_COST);
 7515   format %{ "stlr  zr, $mem\t# int" %}
 7516 
 7517   ins_encode(aarch64_enc_stlr0(mem));
 7518 
 7519   ins_pipe(pipe_class_memory);
 7520 %}
 7521 
 7522 // Store Pointer
 7523 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7524 %{
 7525   match(Set mem (StoreP mem src));
 7526   predicate(n->as_Store()->barrier_data() == 0);
 7527 
 7528   ins_cost(VOLATILE_REF_COST);
 7529   format %{ "stlr  $src, $mem\t# ptr" %}
 7530 
 7531   ins_encode(aarch64_enc_stlr(src, mem));
 7532 
 7533   ins_pipe(pipe_class_memory);
 7534 %}
 7535 
 7536 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7537 %{
 7538   match(Set mem (StoreP mem zero));
 7539   predicate(n->as_Store()->barrier_data() == 0);
 7540 
 7541   ins_cost(VOLATILE_REF_COST);
 7542   format %{ "stlr  zr, $mem\t# ptr" %}
 7543 
 7544   ins_encode(aarch64_enc_stlr0(mem));
 7545 
 7546   ins_pipe(pipe_class_memory);
 7547 %}
 7548 
 7549 // Store Compressed Pointer
 7550 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7551 %{
 7552   match(Set mem (StoreN mem src));
 7553   predicate(n->as_Store()->barrier_data() == 0);
 7554 
 7555   ins_cost(VOLATILE_REF_COST);
 7556   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7557 
 7558   ins_encode(aarch64_enc_stlrw(src, mem));
 7559 
 7560   ins_pipe(pipe_class_memory);
 7561 %}
 7562 
 7563 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7564 %{
 7565   match(Set mem (StoreN mem zero));
 7566   predicate(n->as_Store()->barrier_data() == 0);
 7567 
 7568   ins_cost(VOLATILE_REF_COST);
 7569   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7570 
 7571   ins_encode(aarch64_enc_stlrw0(mem));
 7572 
 7573   ins_pipe(pipe_class_memory);
 7574 %}
 7575 
 7576 // Store Float
 7577 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7578 %{
 7579   match(Set mem (StoreF mem src));
 7580 
 7581   ins_cost(VOLATILE_REF_COST);
 7582   format %{ "stlrs  $src, $mem\t# float" %}
 7583 
 7584   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7585 
 7586   ins_pipe(pipe_class_memory);
 7587 %}
 7588 
 7589 // TODO
 7590 // implement storeImmF0 and storeFImmPacked
 7591 
 7592 // Store Double
 7593 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7594 %{
 7595   match(Set mem (StoreD mem src));
 7596 
 7597   ins_cost(VOLATILE_REF_COST);
 7598   format %{ "stlrd  $src, $mem\t# double" %}
 7599 
 7600   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7601 
 7602   ins_pipe(pipe_class_memory);
 7603 %}
 7604 
 7605 //  ---------------- end of volatile loads and stores ----------------
 7606 
 7607 instruct cacheWB(indirect addr)
 7608 %{
 7609   predicate(VM_Version::supports_data_cache_line_flush());
 7610   match(CacheWB addr);
 7611 
 7612   ins_cost(100);
 7613   format %{"cache wb $addr" %}
 7614   ins_encode %{
 7615     assert($addr->index_position() < 0, "should be");
 7616     assert($addr$$disp == 0, "should be");
 7617     __ cache_wb(Address($addr$$base$$Register, 0));
 7618   %}
 7619   ins_pipe(pipe_slow); // XXX
 7620 %}
 7621 
 7622 instruct cacheWBPreSync()
 7623 %{
 7624   predicate(VM_Version::supports_data_cache_line_flush());
 7625   match(CacheWBPreSync);
 7626 
 7627   ins_cost(100);
 7628   format %{"cache wb presync" %}
 7629   ins_encode %{
 7630     __ cache_wbsync(true);
 7631   %}
 7632   ins_pipe(pipe_slow); // XXX
 7633 %}
 7634 
 7635 instruct cacheWBPostSync()
 7636 %{
 7637   predicate(VM_Version::supports_data_cache_line_flush());
 7638   match(CacheWBPostSync);
 7639 
 7640   ins_cost(100);
 7641   format %{"cache wb postsync" %}
 7642   ins_encode %{
 7643     __ cache_wbsync(false);
 7644   %}
 7645   ins_pipe(pipe_slow); // XXX
 7646 %}
 7647 
 7648 // ============================================================================
 7649 // BSWAP Instructions
 7650 
 7651 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7652   match(Set dst (ReverseBytesI src));
 7653 
 7654   ins_cost(INSN_COST);
 7655   format %{ "revw  $dst, $src" %}
 7656 
 7657   ins_encode %{
 7658     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7659   %}
 7660 
 7661   ins_pipe(ialu_reg);
 7662 %}
 7663 
 7664 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7665   match(Set dst (ReverseBytesL src));
 7666 
 7667   ins_cost(INSN_COST);
 7668   format %{ "rev  $dst, $src" %}
 7669 
 7670   ins_encode %{
 7671     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7672   %}
 7673 
 7674   ins_pipe(ialu_reg);
 7675 %}
 7676 
 7677 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7678   match(Set dst (ReverseBytesUS src));
 7679 
 7680   ins_cost(INSN_COST);
 7681   format %{ "rev16w  $dst, $src" %}
 7682 
 7683   ins_encode %{
 7684     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7685   %}
 7686 
 7687   ins_pipe(ialu_reg);
 7688 %}
 7689 
 7690 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7691   match(Set dst (ReverseBytesS src));
 7692 
 7693   ins_cost(INSN_COST);
 7694   format %{ "rev16w  $dst, $src\n\t"
 7695             "sbfmw $dst, $dst, #0, #15" %}
 7696 
 7697   ins_encode %{
 7698     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7699     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7700   %}
 7701 
 7702   ins_pipe(ialu_reg);
 7703 %}
 7704 
 7705 // ============================================================================
 7706 // Zero Count Instructions
 7707 
 7708 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7709   match(Set dst (CountLeadingZerosI src));
 7710 
 7711   ins_cost(INSN_COST);
 7712   format %{ "clzw  $dst, $src" %}
 7713   ins_encode %{
 7714     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7715   %}
 7716 
 7717   ins_pipe(ialu_reg);
 7718 %}
 7719 
 7720 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7721   match(Set dst (CountLeadingZerosL src));
 7722 
 7723   ins_cost(INSN_COST);
 7724   format %{ "clz   $dst, $src" %}
 7725   ins_encode %{
 7726     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7727   %}
 7728 
 7729   ins_pipe(ialu_reg);
 7730 %}
 7731 
 7732 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7733   match(Set dst (CountTrailingZerosI src));
 7734 
 7735   ins_cost(INSN_COST * 2);
 7736   format %{ "rbitw  $dst, $src\n\t"
 7737             "clzw   $dst, $dst" %}
 7738   ins_encode %{
 7739     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7740     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7741   %}
 7742 
 7743   ins_pipe(ialu_reg);
 7744 %}
 7745 
 7746 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7747   match(Set dst (CountTrailingZerosL src));
 7748 
 7749   ins_cost(INSN_COST * 2);
 7750   format %{ "rbit   $dst, $src\n\t"
 7751             "clz    $dst, $dst" %}
 7752   ins_encode %{
 7753     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7754     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7755   %}
 7756 
 7757   ins_pipe(ialu_reg);
 7758 %}
 7759 
 7760 //---------- Population Count Instructions -------------------------------------
 7761 //
 7762 
 7763 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7764   match(Set dst (PopCountI src));
 7765   effect(TEMP tmp);
 7766   ins_cost(INSN_COST * 13);
 7767 
 7768   format %{ "movw   $src, $src\n\t"
 7769             "mov    $tmp, $src\t# vector (1D)\n\t"
 7770             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7771             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7772             "mov    $dst, $tmp\t# vector (1D)" %}
 7773   ins_encode %{
 7774     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 7775     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7776     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7777     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7778     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7779   %}
 7780 
 7781   ins_pipe(pipe_class_default);
 7782 %}
 7783 
 7784 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7785   match(Set dst (PopCountI (LoadI mem)));
 7786   effect(TEMP tmp);
 7787   ins_cost(INSN_COST * 13);
 7788 
 7789   format %{ "ldrs   $tmp, $mem\n\t"
 7790             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7791             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7792             "mov    $dst, $tmp\t# vector (1D)" %}
 7793   ins_encode %{
 7794     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7795     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7796               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7797     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7798     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7799     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7800   %}
 7801 
 7802   ins_pipe(pipe_class_default);
 7803 %}
 7804 
 7805 // Note: Long.bitCount(long) returns an int.
 7806 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7807   match(Set dst (PopCountL src));
 7808   effect(TEMP tmp);
 7809   ins_cost(INSN_COST * 13);
 7810 
 7811   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7812             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7813             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7814             "mov    $dst, $tmp\t# vector (1D)" %}
 7815   ins_encode %{
 7816     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7817     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7818     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7819     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7820   %}
 7821 
 7822   ins_pipe(pipe_class_default);
 7823 %}
 7824 
 7825 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7826   match(Set dst (PopCountL (LoadL mem)));
 7827   effect(TEMP tmp);
 7828   ins_cost(INSN_COST * 13);
 7829 
 7830   format %{ "ldrd   $tmp, $mem\n\t"
 7831             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7832             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7833             "mov    $dst, $tmp\t# vector (1D)" %}
 7834   ins_encode %{
 7835     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7836     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7837               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7838     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7839     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7840     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7841   %}
 7842 
 7843   ins_pipe(pipe_class_default);
 7844 %}
 7845 
 7846 // ============================================================================
 7847 // VerifyVectorAlignment Instruction
 7848 
 7849 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7850   match(Set addr (VerifyVectorAlignment addr mask));
 7851   effect(KILL cr);
 7852   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7853   ins_encode %{
 7854     Label Lskip;
 7855     // check if masked bits of addr are zero
 7856     __ tst($addr$$Register, $mask$$constant);
 7857     __ br(Assembler::EQ, Lskip);
 7858     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7859     __ bind(Lskip);
 7860   %}
 7861   ins_pipe(pipe_slow);
 7862 %}
 7863 
 7864 // ============================================================================
 7865 // MemBar Instruction
 7866 
 7867 instruct load_fence() %{
 7868   match(LoadFence);
 7869   ins_cost(VOLATILE_REF_COST);
 7870 
 7871   format %{ "load_fence" %}
 7872 
 7873   ins_encode %{
 7874     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7875   %}
 7876   ins_pipe(pipe_serial);
 7877 %}
 7878 
 7879 instruct unnecessary_membar_acquire() %{
 7880   predicate(unnecessary_acquire(n));
 7881   match(MemBarAcquire);
 7882   ins_cost(0);
 7883 
 7884   format %{ "membar_acquire (elided)" %}
 7885 
 7886   ins_encode %{
 7887     __ block_comment("membar_acquire (elided)");
 7888   %}
 7889 
 7890   ins_pipe(pipe_class_empty);
 7891 %}
 7892 
 7893 instruct membar_acquire() %{
 7894   match(MemBarAcquire);
 7895   ins_cost(VOLATILE_REF_COST);
 7896 
 7897   format %{ "membar_acquire\n\t"
 7898             "dmb ishld" %}
 7899 
 7900   ins_encode %{
 7901     __ block_comment("membar_acquire");
 7902     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7903   %}
 7904 
 7905   ins_pipe(pipe_serial);
 7906 %}
 7907 
 7908 
 7909 instruct membar_acquire_lock() %{
 7910   match(MemBarAcquireLock);
 7911   ins_cost(VOLATILE_REF_COST);
 7912 
 7913   format %{ "membar_acquire_lock (elided)" %}
 7914 
 7915   ins_encode %{
 7916     __ block_comment("membar_acquire_lock (elided)");
 7917   %}
 7918 
 7919   ins_pipe(pipe_serial);
 7920 %}
 7921 
 7922 instruct store_fence() %{
 7923   match(StoreFence);
 7924   ins_cost(VOLATILE_REF_COST);
 7925 
 7926   format %{ "store_fence" %}
 7927 
 7928   ins_encode %{
 7929     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 7930   %}
 7931   ins_pipe(pipe_serial);
 7932 %}
 7933 
 7934 instruct unnecessary_membar_release() %{
 7935   predicate(unnecessary_release(n));
 7936   match(MemBarRelease);
 7937   ins_cost(0);
 7938 
 7939   format %{ "membar_release (elided)" %}
 7940 
 7941   ins_encode %{
 7942     __ block_comment("membar_release (elided)");
 7943   %}
 7944   ins_pipe(pipe_serial);
 7945 %}
 7946 
 7947 instruct membar_release() %{
 7948   match(MemBarRelease);
 7949   ins_cost(VOLATILE_REF_COST);
 7950 
 7951   format %{ "membar_release\n\t"
 7952             "dmb ishst\n\tdmb ishld" %}
 7953 
 7954   ins_encode %{
 7955     __ block_comment("membar_release");
 7956     // These will be merged if AlwaysMergeDMB is enabled.
 7957     __ membar(Assembler::StoreStore);
 7958     __ membar(Assembler::LoadStore);
 7959   %}
 7960   ins_pipe(pipe_serial);
 7961 %}
 7962 
 7963 instruct membar_storestore() %{
 7964   match(MemBarStoreStore);
 7965   match(StoreStoreFence);
 7966   ins_cost(VOLATILE_REF_COST);
 7967 
 7968   format %{ "MEMBAR-store-store" %}
 7969 
 7970   ins_encode %{
 7971     __ membar(Assembler::StoreStore);
 7972   %}
 7973   ins_pipe(pipe_serial);
 7974 %}
 7975 
 7976 instruct membar_release_lock() %{
 7977   match(MemBarReleaseLock);
 7978   ins_cost(VOLATILE_REF_COST);
 7979 
 7980   format %{ "membar_release_lock (elided)" %}
 7981 
 7982   ins_encode %{
 7983     __ block_comment("membar_release_lock (elided)");
 7984   %}
 7985 
 7986   ins_pipe(pipe_serial);
 7987 %}
 7988 
 7989 instruct unnecessary_membar_volatile() %{
 7990   predicate(unnecessary_volatile(n));
 7991   match(MemBarVolatile);
 7992   ins_cost(0);
 7993 
 7994   format %{ "membar_volatile (elided)" %}
 7995 
 7996   ins_encode %{
 7997     __ block_comment("membar_volatile (elided)");
 7998   %}
 7999 
 8000   ins_pipe(pipe_serial);
 8001 %}
 8002 
 8003 instruct membar_volatile() %{
 8004   match(MemBarVolatile);
 8005   ins_cost(VOLATILE_REF_COST*100);
 8006 
 8007   format %{ "membar_volatile\n\t"
 8008              "dmb ish"%}
 8009 
 8010   ins_encode %{
 8011     __ block_comment("membar_volatile");
 8012     __ membar(Assembler::StoreLoad);
 8013   %}
 8014 
 8015   ins_pipe(pipe_serial);
 8016 %}
 8017 
 8018 // ============================================================================
 8019 // Cast/Convert Instructions
 8020 
 8021 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8022   match(Set dst (CastX2P src));
 8023 
 8024   ins_cost(INSN_COST);
 8025   format %{ "mov $dst, $src\t# long -> ptr" %}
 8026 
 8027   ins_encode %{
 8028     if ($dst$$reg != $src$$reg) {
 8029       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8030     }
 8031   %}
 8032 
 8033   ins_pipe(ialu_reg);
 8034 %}
 8035 
 8036 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8037   match(Set dst (CastP2X src));
 8038 
 8039   ins_cost(INSN_COST);
 8040   format %{ "mov $dst, $src\t# ptr -> long" %}
 8041 
 8042   ins_encode %{
 8043     if ($dst$$reg != $src$$reg) {
 8044       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8045     }
 8046   %}
 8047 
 8048   ins_pipe(ialu_reg);
 8049 %}
 8050 
 8051 // Convert oop into int for vectors alignment masking
 8052 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8053   match(Set dst (ConvL2I (CastP2X src)));
 8054 
 8055   ins_cost(INSN_COST);
 8056   format %{ "movw $dst, $src\t# ptr -> int" %}
 8057   ins_encode %{
 8058     __ movw($dst$$Register, $src$$Register);
 8059   %}
 8060 
 8061   ins_pipe(ialu_reg);
 8062 %}
 8063 
 8064 // Convert compressed oop into int for vectors alignment masking
 8065 // in case of 32bit oops (heap < 4Gb).
 8066 instruct convN2I(iRegINoSp dst, iRegN src)
 8067 %{
 8068   predicate(CompressedOops::shift() == 0);
 8069   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8070 
 8071   ins_cost(INSN_COST);
 8072   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8073   ins_encode %{
 8074     __ movw($dst$$Register, $src$$Register);
 8075   %}
 8076 
 8077   ins_pipe(ialu_reg);
 8078 %}
 8079 
 8080 
 8081 // Convert oop pointer into compressed form
 8082 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8083   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8084   match(Set dst (EncodeP src));
 8085   effect(KILL cr);
 8086   ins_cost(INSN_COST * 3);
 8087   format %{ "encode_heap_oop $dst, $src" %}
 8088   ins_encode %{
 8089     Register s = $src$$Register;
 8090     Register d = $dst$$Register;
 8091     __ encode_heap_oop(d, s);
 8092   %}
 8093   ins_pipe(ialu_reg);
 8094 %}
 8095 
 8096 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8097   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8098   match(Set dst (EncodeP src));
 8099   ins_cost(INSN_COST * 3);
 8100   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8101   ins_encode %{
 8102     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8103   %}
 8104   ins_pipe(ialu_reg);
 8105 %}
 8106 
 8107 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8108   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8109             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8110   match(Set dst (DecodeN src));
 8111   ins_cost(INSN_COST * 3);
 8112   format %{ "decode_heap_oop $dst, $src" %}
 8113   ins_encode %{
 8114     Register s = $src$$Register;
 8115     Register d = $dst$$Register;
 8116     __ decode_heap_oop(d, s);
 8117   %}
 8118   ins_pipe(ialu_reg);
 8119 %}
 8120 
 8121 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8122   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8123             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8124   match(Set dst (DecodeN src));
 8125   ins_cost(INSN_COST * 3);
 8126   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8127   ins_encode %{
 8128     Register s = $src$$Register;
 8129     Register d = $dst$$Register;
 8130     __ decode_heap_oop_not_null(d, s);
 8131   %}
 8132   ins_pipe(ialu_reg);
 8133 %}
 8134 
 8135 // n.b. AArch64 implementations of encode_klass_not_null and
 8136 // decode_klass_not_null do not modify the flags register so, unlike
 8137 // Intel, we don't kill CR as a side effect here
 8138 
 8139 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8140   match(Set dst (EncodePKlass src));
 8141 
 8142   ins_cost(INSN_COST * 3);
 8143   format %{ "encode_klass_not_null $dst,$src" %}
 8144 
 8145   ins_encode %{
 8146     Register src_reg = as_Register($src$$reg);
 8147     Register dst_reg = as_Register($dst$$reg);
 8148     __ encode_klass_not_null(dst_reg, src_reg);
 8149   %}
 8150 
 8151    ins_pipe(ialu_reg);
 8152 %}
 8153 
 8154 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8155   match(Set dst (DecodeNKlass src));
 8156 
 8157   ins_cost(INSN_COST * 3);
 8158   format %{ "decode_klass_not_null $dst,$src" %}
 8159 
 8160   ins_encode %{
 8161     Register src_reg = as_Register($src$$reg);
 8162     Register dst_reg = as_Register($dst$$reg);
 8163     if (dst_reg != src_reg) {
 8164       __ decode_klass_not_null(dst_reg, src_reg);
 8165     } else {
 8166       __ decode_klass_not_null(dst_reg);
 8167     }
 8168   %}
 8169 
 8170    ins_pipe(ialu_reg);
 8171 %}
 8172 
 8173 instruct checkCastPP(iRegPNoSp dst)
 8174 %{
 8175   match(Set dst (CheckCastPP dst));
 8176 
 8177   size(0);
 8178   format %{ "# checkcastPP of $dst" %}
 8179   ins_encode(/* empty encoding */);
 8180   ins_pipe(pipe_class_empty);
 8181 %}
 8182 
 8183 instruct castPP(iRegPNoSp dst)
 8184 %{
 8185   match(Set dst (CastPP dst));
 8186 
 8187   size(0);
 8188   format %{ "# castPP of $dst" %}
 8189   ins_encode(/* empty encoding */);
 8190   ins_pipe(pipe_class_empty);
 8191 %}
 8192 
 8193 instruct castII(iRegI dst)
 8194 %{
 8195   predicate(VerifyConstraintCasts == 0);
 8196   match(Set dst (CastII dst));
 8197 
 8198   size(0);
 8199   format %{ "# castII of $dst" %}
 8200   ins_encode(/* empty encoding */);
 8201   ins_cost(0);
 8202   ins_pipe(pipe_class_empty);
 8203 %}
 8204 
 8205 instruct castII_checked(iRegI dst, rFlagsReg cr)
 8206 %{
 8207   predicate(VerifyConstraintCasts > 0);
 8208   match(Set dst (CastII dst));
 8209   effect(KILL cr);
 8210 
 8211   format %{ "# castII_checked of $dst" %}
 8212   ins_encode %{
 8213     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
 8214   %}
 8215   ins_pipe(pipe_slow);
 8216 %}
 8217 
 8218 instruct castLL(iRegL dst)
 8219 %{
 8220   predicate(VerifyConstraintCasts == 0);
 8221   match(Set dst (CastLL dst));
 8222 
 8223   size(0);
 8224   format %{ "# castLL of $dst" %}
 8225   ins_encode(/* empty encoding */);
 8226   ins_cost(0);
 8227   ins_pipe(pipe_class_empty);
 8228 %}
 8229 
 8230 instruct castLL_checked(iRegL dst, rFlagsReg cr)
 8231 %{
 8232   predicate(VerifyConstraintCasts > 0);
 8233   match(Set dst (CastLL dst));
 8234   effect(KILL cr);
 8235 
 8236   format %{ "# castLL_checked of $dst" %}
 8237   ins_encode %{
 8238     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
 8239   %}
 8240   ins_pipe(pipe_slow);
 8241 %}
 8242 
 8243 instruct castFF(vRegF dst)
 8244 %{
 8245   match(Set dst (CastFF dst));
 8246 
 8247   size(0);
 8248   format %{ "# castFF of $dst" %}
 8249   ins_encode(/* empty encoding */);
 8250   ins_cost(0);
 8251   ins_pipe(pipe_class_empty);
 8252 %}
 8253 
 8254 instruct castDD(vRegD dst)
 8255 %{
 8256   match(Set dst (CastDD dst));
 8257 
 8258   size(0);
 8259   format %{ "# castDD of $dst" %}
 8260   ins_encode(/* empty encoding */);
 8261   ins_cost(0);
 8262   ins_pipe(pipe_class_empty);
 8263 %}
 8264 
 8265 instruct castVV(vReg dst)
 8266 %{
 8267   match(Set dst (CastVV dst));
 8268 
 8269   size(0);
 8270   format %{ "# castVV of $dst" %}
 8271   ins_encode(/* empty encoding */);
 8272   ins_cost(0);
 8273   ins_pipe(pipe_class_empty);
 8274 %}
 8275 
 8276 instruct castVVMask(pRegGov dst)
 8277 %{
 8278   match(Set dst (CastVV dst));
 8279 
 8280   size(0);
 8281   format %{ "# castVV of $dst" %}
 8282   ins_encode(/* empty encoding */);
 8283   ins_cost(0);
 8284   ins_pipe(pipe_class_empty);
 8285 %}
 8286 
 8287 // ============================================================================
 8288 // Atomic operation instructions
 8289 //
 8290 
 8291 // standard CompareAndSwapX when we are using barriers
 8292 // these have higher priority than the rules selected by a predicate
 8293 
 8294 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8295 // can't match them
 8296 
 8297 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8298 
 8299   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8300   ins_cost(2 * VOLATILE_REF_COST);
 8301 
 8302   effect(KILL cr);
 8303 
 8304   format %{
 8305     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8306     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8307   %}
 8308 
 8309   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8310             aarch64_enc_cset_eq(res));
 8311 
 8312   ins_pipe(pipe_slow);
 8313 %}
 8314 
 8315 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8316 
 8317   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8318   ins_cost(2 * VOLATILE_REF_COST);
 8319 
 8320   effect(KILL cr);
 8321 
 8322   format %{
 8323     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8324     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8325   %}
 8326 
 8327   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8328             aarch64_enc_cset_eq(res));
 8329 
 8330   ins_pipe(pipe_slow);
 8331 %}
 8332 
 8333 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8334 
 8335   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8336   ins_cost(2 * VOLATILE_REF_COST);
 8337 
 8338   effect(KILL cr);
 8339 
 8340  format %{
 8341     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8342     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8343  %}
 8344 
 8345  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8346             aarch64_enc_cset_eq(res));
 8347 
 8348   ins_pipe(pipe_slow);
 8349 %}
 8350 
 8351 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8352 
 8353   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8354   ins_cost(2 * VOLATILE_REF_COST);
 8355 
 8356   effect(KILL cr);
 8357 
 8358  format %{
 8359     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8360     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8361  %}
 8362 
 8363  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8364             aarch64_enc_cset_eq(res));
 8365 
 8366   ins_pipe(pipe_slow);
 8367 %}
 8368 
 8369 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8370 
 8371   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8372   predicate(n->as_LoadStore()->barrier_data() == 0);
 8373   ins_cost(2 * VOLATILE_REF_COST);
 8374 
 8375   effect(KILL cr);
 8376 
 8377  format %{
 8378     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8379     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8380  %}
 8381 
 8382  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8383             aarch64_enc_cset_eq(res));
 8384 
 8385   ins_pipe(pipe_slow);
 8386 %}
 8387 
 8388 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8389 
 8390   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8391   predicate(n->as_LoadStore()->barrier_data() == 0);
 8392   ins_cost(2 * VOLATILE_REF_COST);
 8393 
 8394   effect(KILL cr);
 8395 
 8396  format %{
 8397     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8398     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8399  %}
 8400 
 8401  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8402             aarch64_enc_cset_eq(res));
 8403 
 8404   ins_pipe(pipe_slow);
 8405 %}
 8406 
 8407 // alternative CompareAndSwapX when we are eliding barriers
 8408 
 8409 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8410 
 8411   predicate(needs_acquiring_load_exclusive(n));
 8412   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8413   ins_cost(VOLATILE_REF_COST);
 8414 
 8415   effect(KILL cr);
 8416 
 8417   format %{
 8418     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8419     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8420   %}
 8421 
 8422   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8423             aarch64_enc_cset_eq(res));
 8424 
 8425   ins_pipe(pipe_slow);
 8426 %}
 8427 
 8428 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8429 
 8430   predicate(needs_acquiring_load_exclusive(n));
 8431   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8432   ins_cost(VOLATILE_REF_COST);
 8433 
 8434   effect(KILL cr);
 8435 
 8436   format %{
 8437     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8438     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8439   %}
 8440 
 8441   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8442             aarch64_enc_cset_eq(res));
 8443 
 8444   ins_pipe(pipe_slow);
 8445 %}
 8446 
 8447 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8448 
 8449   predicate(needs_acquiring_load_exclusive(n));
 8450   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8451   ins_cost(VOLATILE_REF_COST);
 8452 
 8453   effect(KILL cr);
 8454 
 8455  format %{
 8456     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8457     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8458  %}
 8459 
 8460  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8461             aarch64_enc_cset_eq(res));
 8462 
 8463   ins_pipe(pipe_slow);
 8464 %}
 8465 
 8466 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8467 
 8468   predicate(needs_acquiring_load_exclusive(n));
 8469   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8470   ins_cost(VOLATILE_REF_COST);
 8471 
 8472   effect(KILL cr);
 8473 
 8474  format %{
 8475     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8476     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8477  %}
 8478 
 8479  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8480             aarch64_enc_cset_eq(res));
 8481 
 8482   ins_pipe(pipe_slow);
 8483 %}
 8484 
 8485 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8486 
 8487   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8488   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8489   ins_cost(VOLATILE_REF_COST);
 8490 
 8491   effect(KILL cr);
 8492 
 8493  format %{
 8494     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8495     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8496  %}
 8497 
 8498  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8499             aarch64_enc_cset_eq(res));
 8500 
 8501   ins_pipe(pipe_slow);
 8502 %}
 8503 
 8504 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8505 
 8506   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8507   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8508   ins_cost(VOLATILE_REF_COST);
 8509 
 8510   effect(KILL cr);
 8511 
 8512  format %{
 8513     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8514     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8515  %}
 8516 
 8517  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8518             aarch64_enc_cset_eq(res));
 8519 
 8520   ins_pipe(pipe_slow);
 8521 %}
 8522 
 8523 
 8524 // ---------------------------------------------------------------------
 8525 
 8526 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8527 
 8528 // Sundry CAS operations.  Note that release is always true,
 8529 // regardless of the memory ordering of the CAS.  This is because we
 8530 // need the volatile case to be sequentially consistent but there is
 8531 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8532 // can't check the type of memory ordering here, so we always emit a
 8533 // STLXR.
 8534 
 8535 // This section is generated from cas.m4
 8536 
 8537 
 8538 // This pattern is generated automatically from cas.m4.
 8539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8540 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8541   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8542   ins_cost(2 * VOLATILE_REF_COST);
 8543   effect(TEMP_DEF res, KILL cr);
 8544   format %{
 8545     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8546   %}
 8547   ins_encode %{
 8548     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8549                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8550                /*weak*/ false, $res$$Register);
 8551     __ sxtbw($res$$Register, $res$$Register);
 8552   %}
 8553   ins_pipe(pipe_slow);
 8554 %}
 8555 
 8556 // This pattern is generated automatically from cas.m4.
 8557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8558 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8559   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8560   ins_cost(2 * VOLATILE_REF_COST);
 8561   effect(TEMP_DEF res, KILL cr);
 8562   format %{
 8563     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8564   %}
 8565   ins_encode %{
 8566     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8567                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8568                /*weak*/ false, $res$$Register);
 8569     __ sxthw($res$$Register, $res$$Register);
 8570   %}
 8571   ins_pipe(pipe_slow);
 8572 %}
 8573 
 8574 // This pattern is generated automatically from cas.m4.
 8575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8576 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8577   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8578   ins_cost(2 * VOLATILE_REF_COST);
 8579   effect(TEMP_DEF res, KILL cr);
 8580   format %{
 8581     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8582   %}
 8583   ins_encode %{
 8584     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8585                Assembler::word, /*acquire*/ false, /*release*/ true,
 8586                /*weak*/ false, $res$$Register);
 8587   %}
 8588   ins_pipe(pipe_slow);
 8589 %}
 8590 
 8591 // This pattern is generated automatically from cas.m4.
 8592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8593 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8594   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8595   ins_cost(2 * VOLATILE_REF_COST);
 8596   effect(TEMP_DEF res, KILL cr);
 8597   format %{
 8598     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8599   %}
 8600   ins_encode %{
 8601     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8602                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8603                /*weak*/ false, $res$$Register);
 8604   %}
 8605   ins_pipe(pipe_slow);
 8606 %}
 8607 
 8608 // This pattern is generated automatically from cas.m4.
 8609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8610 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8611   predicate(n->as_LoadStore()->barrier_data() == 0);
 8612   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8613   ins_cost(2 * VOLATILE_REF_COST);
 8614   effect(TEMP_DEF res, KILL cr);
 8615   format %{
 8616     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8617   %}
 8618   ins_encode %{
 8619     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8620                Assembler::word, /*acquire*/ false, /*release*/ true,
 8621                /*weak*/ false, $res$$Register);
 8622   %}
 8623   ins_pipe(pipe_slow);
 8624 %}
 8625 
 8626 // This pattern is generated automatically from cas.m4.
 8627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8628 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8629   predicate(n->as_LoadStore()->barrier_data() == 0);
 8630   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8631   ins_cost(2 * VOLATILE_REF_COST);
 8632   effect(TEMP_DEF res, KILL cr);
 8633   format %{
 8634     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8635   %}
 8636   ins_encode %{
 8637     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8638                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8639                /*weak*/ false, $res$$Register);
 8640   %}
 8641   ins_pipe(pipe_slow);
 8642 %}
 8643 
 8644 // This pattern is generated automatically from cas.m4.
 8645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8646 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8647   predicate(needs_acquiring_load_exclusive(n));
 8648   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8649   ins_cost(VOLATILE_REF_COST);
 8650   effect(TEMP_DEF res, KILL cr);
 8651   format %{
 8652     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8653   %}
 8654   ins_encode %{
 8655     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8656                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8657                /*weak*/ false, $res$$Register);
 8658     __ sxtbw($res$$Register, $res$$Register);
 8659   %}
 8660   ins_pipe(pipe_slow);
 8661 %}
 8662 
 8663 // This pattern is generated automatically from cas.m4.
 8664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8665 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8666   predicate(needs_acquiring_load_exclusive(n));
 8667   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8668   ins_cost(VOLATILE_REF_COST);
 8669   effect(TEMP_DEF res, KILL cr);
 8670   format %{
 8671     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8672   %}
 8673   ins_encode %{
 8674     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8675                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8676                /*weak*/ false, $res$$Register);
 8677     __ sxthw($res$$Register, $res$$Register);
 8678   %}
 8679   ins_pipe(pipe_slow);
 8680 %}
 8681 
 8682 // This pattern is generated automatically from cas.m4.
 8683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8684 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8685   predicate(needs_acquiring_load_exclusive(n));
 8686   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8687   ins_cost(VOLATILE_REF_COST);
 8688   effect(TEMP_DEF res, KILL cr);
 8689   format %{
 8690     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8691   %}
 8692   ins_encode %{
 8693     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8694                Assembler::word, /*acquire*/ true, /*release*/ true,
 8695                /*weak*/ false, $res$$Register);
 8696   %}
 8697   ins_pipe(pipe_slow);
 8698 %}
 8699 
 8700 // This pattern is generated automatically from cas.m4.
 8701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8702 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8703   predicate(needs_acquiring_load_exclusive(n));
 8704   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8705   ins_cost(VOLATILE_REF_COST);
 8706   effect(TEMP_DEF res, KILL cr);
 8707   format %{
 8708     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8709   %}
 8710   ins_encode %{
 8711     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8712                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8713                /*weak*/ false, $res$$Register);
 8714   %}
 8715   ins_pipe(pipe_slow);
 8716 %}
 8717 
 8718 // This pattern is generated automatically from cas.m4.
 8719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8720 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8721   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8722   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8723   ins_cost(VOLATILE_REF_COST);
 8724   effect(TEMP_DEF res, KILL cr);
 8725   format %{
 8726     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8727   %}
 8728   ins_encode %{
 8729     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8730                Assembler::word, /*acquire*/ true, /*release*/ true,
 8731                /*weak*/ false, $res$$Register);
 8732   %}
 8733   ins_pipe(pipe_slow);
 8734 %}
 8735 
 8736 // This pattern is generated automatically from cas.m4.
 8737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8738 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8739   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8740   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8741   ins_cost(VOLATILE_REF_COST);
 8742   effect(TEMP_DEF res, KILL cr);
 8743   format %{
 8744     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8745   %}
 8746   ins_encode %{
 8747     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8748                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8749                /*weak*/ false, $res$$Register);
 8750   %}
 8751   ins_pipe(pipe_slow);
 8752 %}
 8753 
 8754 // This pattern is generated automatically from cas.m4.
 8755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8756 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8757   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8758   ins_cost(2 * VOLATILE_REF_COST);
 8759   effect(KILL cr);
 8760   format %{
 8761     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8762     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8763   %}
 8764   ins_encode %{
 8765     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8766                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8767                /*weak*/ true, noreg);
 8768     __ csetw($res$$Register, Assembler::EQ);
 8769   %}
 8770   ins_pipe(pipe_slow);
 8771 %}
 8772 
 8773 // This pattern is generated automatically from cas.m4.
 8774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8775 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8776   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8777   ins_cost(2 * VOLATILE_REF_COST);
 8778   effect(KILL cr);
 8779   format %{
 8780     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8781     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8782   %}
 8783   ins_encode %{
 8784     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8785                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8786                /*weak*/ true, noreg);
 8787     __ csetw($res$$Register, Assembler::EQ);
 8788   %}
 8789   ins_pipe(pipe_slow);
 8790 %}
 8791 
 8792 // This pattern is generated automatically from cas.m4.
 8793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8794 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8795   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8796   ins_cost(2 * VOLATILE_REF_COST);
 8797   effect(KILL cr);
 8798   format %{
 8799     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8800     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8801   %}
 8802   ins_encode %{
 8803     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8804                Assembler::word, /*acquire*/ false, /*release*/ true,
 8805                /*weak*/ true, noreg);
 8806     __ csetw($res$$Register, Assembler::EQ);
 8807   %}
 8808   ins_pipe(pipe_slow);
 8809 %}
 8810 
 8811 // This pattern is generated automatically from cas.m4.
 8812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8813 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8814   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8815   ins_cost(2 * VOLATILE_REF_COST);
 8816   effect(KILL cr);
 8817   format %{
 8818     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8819     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8820   %}
 8821   ins_encode %{
 8822     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8823                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8824                /*weak*/ true, noreg);
 8825     __ csetw($res$$Register, Assembler::EQ);
 8826   %}
 8827   ins_pipe(pipe_slow);
 8828 %}
 8829 
 8830 // This pattern is generated automatically from cas.m4.
 8831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8832 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8833   predicate(n->as_LoadStore()->barrier_data() == 0);
 8834   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8835   ins_cost(2 * VOLATILE_REF_COST);
 8836   effect(KILL cr);
 8837   format %{
 8838     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8839     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8840   %}
 8841   ins_encode %{
 8842     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8843                Assembler::word, /*acquire*/ false, /*release*/ true,
 8844                /*weak*/ true, noreg);
 8845     __ csetw($res$$Register, Assembler::EQ);
 8846   %}
 8847   ins_pipe(pipe_slow);
 8848 %}
 8849 
 8850 // This pattern is generated automatically from cas.m4.
 8851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8852 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8853   predicate(n->as_LoadStore()->barrier_data() == 0);
 8854   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8855   ins_cost(2 * VOLATILE_REF_COST);
 8856   effect(KILL cr);
 8857   format %{
 8858     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8859     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8860   %}
 8861   ins_encode %{
 8862     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8863                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8864                /*weak*/ true, noreg);
 8865     __ csetw($res$$Register, Assembler::EQ);
 8866   %}
 8867   ins_pipe(pipe_slow);
 8868 %}
 8869 
 8870 // This pattern is generated automatically from cas.m4.
 8871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8872 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8873   predicate(needs_acquiring_load_exclusive(n));
 8874   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8875   ins_cost(VOLATILE_REF_COST);
 8876   effect(KILL cr);
 8877   format %{
 8878     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8879     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8880   %}
 8881   ins_encode %{
 8882     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8883                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8884                /*weak*/ true, noreg);
 8885     __ csetw($res$$Register, Assembler::EQ);
 8886   %}
 8887   ins_pipe(pipe_slow);
 8888 %}
 8889 
 8890 // This pattern is generated automatically from cas.m4.
 8891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8892 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8893   predicate(needs_acquiring_load_exclusive(n));
 8894   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8895   ins_cost(VOLATILE_REF_COST);
 8896   effect(KILL cr);
 8897   format %{
 8898     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8899     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8900   %}
 8901   ins_encode %{
 8902     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8903                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8904                /*weak*/ true, noreg);
 8905     __ csetw($res$$Register, Assembler::EQ);
 8906   %}
 8907   ins_pipe(pipe_slow);
 8908 %}
 8909 
 8910 // This pattern is generated automatically from cas.m4.
 8911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8912 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8913   predicate(needs_acquiring_load_exclusive(n));
 8914   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8915   ins_cost(VOLATILE_REF_COST);
 8916   effect(KILL cr);
 8917   format %{
 8918     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8919     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8920   %}
 8921   ins_encode %{
 8922     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8923                Assembler::word, /*acquire*/ true, /*release*/ true,
 8924                /*weak*/ true, noreg);
 8925     __ csetw($res$$Register, Assembler::EQ);
 8926   %}
 8927   ins_pipe(pipe_slow);
 8928 %}
 8929 
 8930 // This pattern is generated automatically from cas.m4.
 8931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8932 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8933   predicate(needs_acquiring_load_exclusive(n));
 8934   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8935   ins_cost(VOLATILE_REF_COST);
 8936   effect(KILL cr);
 8937   format %{
 8938     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8939     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8940   %}
 8941   ins_encode %{
 8942     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8943                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8944                /*weak*/ true, noreg);
 8945     __ csetw($res$$Register, Assembler::EQ);
 8946   %}
 8947   ins_pipe(pipe_slow);
 8948 %}
 8949 
 8950 // This pattern is generated automatically from cas.m4.
 8951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8952 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8953   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8954   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8955   ins_cost(VOLATILE_REF_COST);
 8956   effect(KILL cr);
 8957   format %{
 8958     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8959     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8960   %}
 8961   ins_encode %{
 8962     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8963                Assembler::word, /*acquire*/ true, /*release*/ true,
 8964                /*weak*/ true, noreg);
 8965     __ csetw($res$$Register, Assembler::EQ);
 8966   %}
 8967   ins_pipe(pipe_slow);
 8968 %}
 8969 
 8970 // This pattern is generated automatically from cas.m4.
 8971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8972 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8973   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8974   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8975   ins_cost(VOLATILE_REF_COST);
 8976   effect(KILL cr);
 8977   format %{
 8978     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8979     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8980   %}
 8981   ins_encode %{
 8982     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8983                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8984                /*weak*/ true, noreg);
 8985     __ csetw($res$$Register, Assembler::EQ);
 8986   %}
 8987   ins_pipe(pipe_slow);
 8988 %}
 8989 
 8990 // END This section of the file is automatically generated. Do not edit --------------
 8991 // ---------------------------------------------------------------------
 8992 
 8993 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 8994   match(Set prev (GetAndSetI mem newv));
 8995   ins_cost(2 * VOLATILE_REF_COST);
 8996   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 8997   ins_encode %{
 8998     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8999   %}
 9000   ins_pipe(pipe_serial);
 9001 %}
 9002 
 9003 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9004   match(Set prev (GetAndSetL mem newv));
 9005   ins_cost(2 * VOLATILE_REF_COST);
 9006   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9007   ins_encode %{
 9008     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9009   %}
 9010   ins_pipe(pipe_serial);
 9011 %}
 9012 
 9013 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9014   predicate(n->as_LoadStore()->barrier_data() == 0);
 9015   match(Set prev (GetAndSetN mem newv));
 9016   ins_cost(2 * VOLATILE_REF_COST);
 9017   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9018   ins_encode %{
 9019     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9020   %}
 9021   ins_pipe(pipe_serial);
 9022 %}
 9023 
 9024 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9025   predicate(n->as_LoadStore()->barrier_data() == 0);
 9026   match(Set prev (GetAndSetP mem newv));
 9027   ins_cost(2 * VOLATILE_REF_COST);
 9028   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9029   ins_encode %{
 9030     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9031   %}
 9032   ins_pipe(pipe_serial);
 9033 %}
 9034 
 9035 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9036   predicate(needs_acquiring_load_exclusive(n));
 9037   match(Set prev (GetAndSetI mem newv));
 9038   ins_cost(VOLATILE_REF_COST);
 9039   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9040   ins_encode %{
 9041     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9042   %}
 9043   ins_pipe(pipe_serial);
 9044 %}
 9045 
 9046 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9047   predicate(needs_acquiring_load_exclusive(n));
 9048   match(Set prev (GetAndSetL mem newv));
 9049   ins_cost(VOLATILE_REF_COST);
 9050   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9051   ins_encode %{
 9052     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9053   %}
 9054   ins_pipe(pipe_serial);
 9055 %}
 9056 
 9057 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9058   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9059   match(Set prev (GetAndSetN mem newv));
 9060   ins_cost(VOLATILE_REF_COST);
 9061   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9062   ins_encode %{
 9063     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9064   %}
 9065   ins_pipe(pipe_serial);
 9066 %}
 9067 
 9068 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9069   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9070   match(Set prev (GetAndSetP mem newv));
 9071   ins_cost(VOLATILE_REF_COST);
 9072   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9073   ins_encode %{
 9074     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9075   %}
 9076   ins_pipe(pipe_serial);
 9077 %}
 9078 
 9079 
 9080 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9081   match(Set newval (GetAndAddL mem incr));
 9082   ins_cost(2 * VOLATILE_REF_COST + 1);
 9083   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9084   ins_encode %{
 9085     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9086   %}
 9087   ins_pipe(pipe_serial);
 9088 %}
 9089 
 9090 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9091   predicate(n->as_LoadStore()->result_not_used());
 9092   match(Set dummy (GetAndAddL mem incr));
 9093   ins_cost(2 * VOLATILE_REF_COST);
 9094   format %{ "get_and_addL [$mem], $incr" %}
 9095   ins_encode %{
 9096     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9097   %}
 9098   ins_pipe(pipe_serial);
 9099 %}
 9100 
 9101 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9102   match(Set newval (GetAndAddL mem incr));
 9103   ins_cost(2 * VOLATILE_REF_COST + 1);
 9104   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9105   ins_encode %{
 9106     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9107   %}
 9108   ins_pipe(pipe_serial);
 9109 %}
 9110 
 9111 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9112   predicate(n->as_LoadStore()->result_not_used());
 9113   match(Set dummy (GetAndAddL mem incr));
 9114   ins_cost(2 * VOLATILE_REF_COST);
 9115   format %{ "get_and_addL [$mem], $incr" %}
 9116   ins_encode %{
 9117     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9118   %}
 9119   ins_pipe(pipe_serial);
 9120 %}
 9121 
 9122 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9123   match(Set newval (GetAndAddI mem incr));
 9124   ins_cost(2 * VOLATILE_REF_COST + 1);
 9125   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9126   ins_encode %{
 9127     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9128   %}
 9129   ins_pipe(pipe_serial);
 9130 %}
 9131 
 9132 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9133   predicate(n->as_LoadStore()->result_not_used());
 9134   match(Set dummy (GetAndAddI mem incr));
 9135   ins_cost(2 * VOLATILE_REF_COST);
 9136   format %{ "get_and_addI [$mem], $incr" %}
 9137   ins_encode %{
 9138     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9139   %}
 9140   ins_pipe(pipe_serial);
 9141 %}
 9142 
 9143 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9144   match(Set newval (GetAndAddI mem incr));
 9145   ins_cost(2 * VOLATILE_REF_COST + 1);
 9146   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9147   ins_encode %{
 9148     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9149   %}
 9150   ins_pipe(pipe_serial);
 9151 %}
 9152 
 9153 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9154   predicate(n->as_LoadStore()->result_not_used());
 9155   match(Set dummy (GetAndAddI mem incr));
 9156   ins_cost(2 * VOLATILE_REF_COST);
 9157   format %{ "get_and_addI [$mem], $incr" %}
 9158   ins_encode %{
 9159     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9160   %}
 9161   ins_pipe(pipe_serial);
 9162 %}
 9163 
 9164 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9165   predicate(needs_acquiring_load_exclusive(n));
 9166   match(Set newval (GetAndAddL mem incr));
 9167   ins_cost(VOLATILE_REF_COST + 1);
 9168   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9169   ins_encode %{
 9170     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9171   %}
 9172   ins_pipe(pipe_serial);
 9173 %}
 9174 
 9175 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9176   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9177   match(Set dummy (GetAndAddL mem incr));
 9178   ins_cost(VOLATILE_REF_COST);
 9179   format %{ "get_and_addL_acq [$mem], $incr" %}
 9180   ins_encode %{
 9181     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9182   %}
 9183   ins_pipe(pipe_serial);
 9184 %}
 9185 
 9186 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9187   predicate(needs_acquiring_load_exclusive(n));
 9188   match(Set newval (GetAndAddL mem incr));
 9189   ins_cost(VOLATILE_REF_COST + 1);
 9190   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9191   ins_encode %{
 9192     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9193   %}
 9194   ins_pipe(pipe_serial);
 9195 %}
 9196 
 9197 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9198   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9199   match(Set dummy (GetAndAddL mem incr));
 9200   ins_cost(VOLATILE_REF_COST);
 9201   format %{ "get_and_addL_acq [$mem], $incr" %}
 9202   ins_encode %{
 9203     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9204   %}
 9205   ins_pipe(pipe_serial);
 9206 %}
 9207 
 9208 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9209   predicate(needs_acquiring_load_exclusive(n));
 9210   match(Set newval (GetAndAddI mem incr));
 9211   ins_cost(VOLATILE_REF_COST + 1);
 9212   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9213   ins_encode %{
 9214     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9215   %}
 9216   ins_pipe(pipe_serial);
 9217 %}
 9218 
 9219 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9220   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9221   match(Set dummy (GetAndAddI mem incr));
 9222   ins_cost(VOLATILE_REF_COST);
 9223   format %{ "get_and_addI_acq [$mem], $incr" %}
 9224   ins_encode %{
 9225     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9226   %}
 9227   ins_pipe(pipe_serial);
 9228 %}
 9229 
 9230 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9231   predicate(needs_acquiring_load_exclusive(n));
 9232   match(Set newval (GetAndAddI mem incr));
 9233   ins_cost(VOLATILE_REF_COST + 1);
 9234   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9235   ins_encode %{
 9236     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9237   %}
 9238   ins_pipe(pipe_serial);
 9239 %}
 9240 
 9241 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9242   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9243   match(Set dummy (GetAndAddI mem incr));
 9244   ins_cost(VOLATILE_REF_COST);
 9245   format %{ "get_and_addI_acq [$mem], $incr" %}
 9246   ins_encode %{
 9247     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9248   %}
 9249   ins_pipe(pipe_serial);
 9250 %}
 9251 
 9252 // Manifest a CmpU result in an integer register.
 9253 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9254 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9255 %{
 9256   match(Set dst (CmpU3 src1 src2));
 9257   effect(KILL flags);
 9258 
 9259   ins_cost(INSN_COST * 3);
 9260   format %{
 9261       "cmpw $src1, $src2\n\t"
 9262       "csetw $dst, ne\n\t"
 9263       "cnegw $dst, lo\t# CmpU3(reg)"
 9264   %}
 9265   ins_encode %{
 9266     __ cmpw($src1$$Register, $src2$$Register);
 9267     __ csetw($dst$$Register, Assembler::NE);
 9268     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9269   %}
 9270 
 9271   ins_pipe(pipe_class_default);
 9272 %}
 9273 
 9274 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9275 %{
 9276   match(Set dst (CmpU3 src1 src2));
 9277   effect(KILL flags);
 9278 
 9279   ins_cost(INSN_COST * 3);
 9280   format %{
 9281       "subsw zr, $src1, $src2\n\t"
 9282       "csetw $dst, ne\n\t"
 9283       "cnegw $dst, lo\t# CmpU3(imm)"
 9284   %}
 9285   ins_encode %{
 9286     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9287     __ csetw($dst$$Register, Assembler::NE);
 9288     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9289   %}
 9290 
 9291   ins_pipe(pipe_class_default);
 9292 %}
 9293 
 9294 // Manifest a CmpUL result in an integer register.
 9295 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9296 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9297 %{
 9298   match(Set dst (CmpUL3 src1 src2));
 9299   effect(KILL flags);
 9300 
 9301   ins_cost(INSN_COST * 3);
 9302   format %{
 9303       "cmp $src1, $src2\n\t"
 9304       "csetw $dst, ne\n\t"
 9305       "cnegw $dst, lo\t# CmpUL3(reg)"
 9306   %}
 9307   ins_encode %{
 9308     __ cmp($src1$$Register, $src2$$Register);
 9309     __ csetw($dst$$Register, Assembler::NE);
 9310     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9311   %}
 9312 
 9313   ins_pipe(pipe_class_default);
 9314 %}
 9315 
 9316 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9317 %{
 9318   match(Set dst (CmpUL3 src1 src2));
 9319   effect(KILL flags);
 9320 
 9321   ins_cost(INSN_COST * 3);
 9322   format %{
 9323       "subs zr, $src1, $src2\n\t"
 9324       "csetw $dst, ne\n\t"
 9325       "cnegw $dst, lo\t# CmpUL3(imm)"
 9326   %}
 9327   ins_encode %{
 9328     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9329     __ csetw($dst$$Register, Assembler::NE);
 9330     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9331   %}
 9332 
 9333   ins_pipe(pipe_class_default);
 9334 %}
 9335 
 9336 // Manifest a CmpL result in an integer register.
 9337 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9338 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9339 %{
 9340   match(Set dst (CmpL3 src1 src2));
 9341   effect(KILL flags);
 9342 
 9343   ins_cost(INSN_COST * 3);
 9344   format %{
 9345       "cmp $src1, $src2\n\t"
 9346       "csetw $dst, ne\n\t"
 9347       "cnegw $dst, lt\t# CmpL3(reg)"
 9348   %}
 9349   ins_encode %{
 9350     __ cmp($src1$$Register, $src2$$Register);
 9351     __ csetw($dst$$Register, Assembler::NE);
 9352     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9353   %}
 9354 
 9355   ins_pipe(pipe_class_default);
 9356 %}
 9357 
 9358 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9359 %{
 9360   match(Set dst (CmpL3 src1 src2));
 9361   effect(KILL flags);
 9362 
 9363   ins_cost(INSN_COST * 3);
 9364   format %{
 9365       "subs zr, $src1, $src2\n\t"
 9366       "csetw $dst, ne\n\t"
 9367       "cnegw $dst, lt\t# CmpL3(imm)"
 9368   %}
 9369   ins_encode %{
 9370     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9371     __ csetw($dst$$Register, Assembler::NE);
 9372     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9373   %}
 9374 
 9375   ins_pipe(pipe_class_default);
 9376 %}
 9377 
 9378 // ============================================================================
 9379 // Conditional Move Instructions
 9380 
 9381 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9382 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9383 // define an op class which merged both inputs and use it to type the
 9384 // argument to a single rule. unfortunatelyt his fails because the
 9385 // opclass does not live up to the COND_INTER interface of its
 9386 // component operands. When the generic code tries to negate the
 9387 // operand it ends up running the generci Machoper::negate method
 9388 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9389 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9390 
 9391 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9392   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9393 
 9394   ins_cost(INSN_COST * 2);
 9395   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9396 
 9397   ins_encode %{
 9398     __ cselw(as_Register($dst$$reg),
 9399              as_Register($src2$$reg),
 9400              as_Register($src1$$reg),
 9401              (Assembler::Condition)$cmp$$cmpcode);
 9402   %}
 9403 
 9404   ins_pipe(icond_reg_reg);
 9405 %}
 9406 
 9407 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9408   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9409 
 9410   ins_cost(INSN_COST * 2);
 9411   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9412 
 9413   ins_encode %{
 9414     __ cselw(as_Register($dst$$reg),
 9415              as_Register($src2$$reg),
 9416              as_Register($src1$$reg),
 9417              (Assembler::Condition)$cmp$$cmpcode);
 9418   %}
 9419 
 9420   ins_pipe(icond_reg_reg);
 9421 %}
 9422 
 9423 // special cases where one arg is zero
 9424 
 9425 // n.b. this is selected in preference to the rule above because it
 9426 // avoids loading constant 0 into a source register
 9427 
 9428 // TODO
 9429 // we ought only to be able to cull one of these variants as the ideal
 9430 // transforms ought always to order the zero consistently (to left/right?)
 9431 
 9432 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9433   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9434 
 9435   ins_cost(INSN_COST * 2);
 9436   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9437 
 9438   ins_encode %{
 9439     __ cselw(as_Register($dst$$reg),
 9440              as_Register($src$$reg),
 9441              zr,
 9442              (Assembler::Condition)$cmp$$cmpcode);
 9443   %}
 9444 
 9445   ins_pipe(icond_reg);
 9446 %}
 9447 
 9448 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9449   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9450 
 9451   ins_cost(INSN_COST * 2);
 9452   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9453 
 9454   ins_encode %{
 9455     __ cselw(as_Register($dst$$reg),
 9456              as_Register($src$$reg),
 9457              zr,
 9458              (Assembler::Condition)$cmp$$cmpcode);
 9459   %}
 9460 
 9461   ins_pipe(icond_reg);
 9462 %}
 9463 
 9464 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9465   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9466 
 9467   ins_cost(INSN_COST * 2);
 9468   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9469 
 9470   ins_encode %{
 9471     __ cselw(as_Register($dst$$reg),
 9472              zr,
 9473              as_Register($src$$reg),
 9474              (Assembler::Condition)$cmp$$cmpcode);
 9475   %}
 9476 
 9477   ins_pipe(icond_reg);
 9478 %}
 9479 
 9480 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9481   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9482 
 9483   ins_cost(INSN_COST * 2);
 9484   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9485 
 9486   ins_encode %{
 9487     __ cselw(as_Register($dst$$reg),
 9488              zr,
 9489              as_Register($src$$reg),
 9490              (Assembler::Condition)$cmp$$cmpcode);
 9491   %}
 9492 
 9493   ins_pipe(icond_reg);
 9494 %}
 9495 
 9496 // special case for creating a boolean 0 or 1
 9497 
 9498 // n.b. this is selected in preference to the rule above because it
 9499 // avoids loading constants 0 and 1 into a source register
 9500 
 9501 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9502   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9503 
 9504   ins_cost(INSN_COST * 2);
 9505   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9506 
 9507   ins_encode %{
 9508     // equivalently
 9509     // cset(as_Register($dst$$reg),
 9510     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9511     __ csincw(as_Register($dst$$reg),
 9512              zr,
 9513              zr,
 9514              (Assembler::Condition)$cmp$$cmpcode);
 9515   %}
 9516 
 9517   ins_pipe(icond_none);
 9518 %}
 9519 
 9520 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9521   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9522 
 9523   ins_cost(INSN_COST * 2);
 9524   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9525 
 9526   ins_encode %{
 9527     // equivalently
 9528     // cset(as_Register($dst$$reg),
 9529     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9530     __ csincw(as_Register($dst$$reg),
 9531              zr,
 9532              zr,
 9533              (Assembler::Condition)$cmp$$cmpcode);
 9534   %}
 9535 
 9536   ins_pipe(icond_none);
 9537 %}
 9538 
 9539 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9540   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9541 
 9542   ins_cost(INSN_COST * 2);
 9543   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9544 
 9545   ins_encode %{
 9546     __ csel(as_Register($dst$$reg),
 9547             as_Register($src2$$reg),
 9548             as_Register($src1$$reg),
 9549             (Assembler::Condition)$cmp$$cmpcode);
 9550   %}
 9551 
 9552   ins_pipe(icond_reg_reg);
 9553 %}
 9554 
 9555 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9556   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9557 
 9558   ins_cost(INSN_COST * 2);
 9559   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9560 
 9561   ins_encode %{
 9562     __ csel(as_Register($dst$$reg),
 9563             as_Register($src2$$reg),
 9564             as_Register($src1$$reg),
 9565             (Assembler::Condition)$cmp$$cmpcode);
 9566   %}
 9567 
 9568   ins_pipe(icond_reg_reg);
 9569 %}
 9570 
 9571 // special cases where one arg is zero
 9572 
 9573 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9574   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9575 
 9576   ins_cost(INSN_COST * 2);
 9577   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9578 
 9579   ins_encode %{
 9580     __ csel(as_Register($dst$$reg),
 9581             zr,
 9582             as_Register($src$$reg),
 9583             (Assembler::Condition)$cmp$$cmpcode);
 9584   %}
 9585 
 9586   ins_pipe(icond_reg);
 9587 %}
 9588 
 9589 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9590   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9591 
 9592   ins_cost(INSN_COST * 2);
 9593   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9594 
 9595   ins_encode %{
 9596     __ csel(as_Register($dst$$reg),
 9597             zr,
 9598             as_Register($src$$reg),
 9599             (Assembler::Condition)$cmp$$cmpcode);
 9600   %}
 9601 
 9602   ins_pipe(icond_reg);
 9603 %}
 9604 
 9605 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9606   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9607 
 9608   ins_cost(INSN_COST * 2);
 9609   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9610 
 9611   ins_encode %{
 9612     __ csel(as_Register($dst$$reg),
 9613             as_Register($src$$reg),
 9614             zr,
 9615             (Assembler::Condition)$cmp$$cmpcode);
 9616   %}
 9617 
 9618   ins_pipe(icond_reg);
 9619 %}
 9620 
 9621 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9622   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9623 
 9624   ins_cost(INSN_COST * 2);
 9625   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9626 
 9627   ins_encode %{
 9628     __ csel(as_Register($dst$$reg),
 9629             as_Register($src$$reg),
 9630             zr,
 9631             (Assembler::Condition)$cmp$$cmpcode);
 9632   %}
 9633 
 9634   ins_pipe(icond_reg);
 9635 %}
 9636 
 9637 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9638   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9639 
 9640   ins_cost(INSN_COST * 2);
 9641   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9642 
 9643   ins_encode %{
 9644     __ csel(as_Register($dst$$reg),
 9645             as_Register($src2$$reg),
 9646             as_Register($src1$$reg),
 9647             (Assembler::Condition)$cmp$$cmpcode);
 9648   %}
 9649 
 9650   ins_pipe(icond_reg_reg);
 9651 %}
 9652 
 9653 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9654   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9655 
 9656   ins_cost(INSN_COST * 2);
 9657   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9658 
 9659   ins_encode %{
 9660     __ csel(as_Register($dst$$reg),
 9661             as_Register($src2$$reg),
 9662             as_Register($src1$$reg),
 9663             (Assembler::Condition)$cmp$$cmpcode);
 9664   %}
 9665 
 9666   ins_pipe(icond_reg_reg);
 9667 %}
 9668 
 9669 // special cases where one arg is zero
 9670 
 9671 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9672   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9673 
 9674   ins_cost(INSN_COST * 2);
 9675   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9676 
 9677   ins_encode %{
 9678     __ csel(as_Register($dst$$reg),
 9679             zr,
 9680             as_Register($src$$reg),
 9681             (Assembler::Condition)$cmp$$cmpcode);
 9682   %}
 9683 
 9684   ins_pipe(icond_reg);
 9685 %}
 9686 
 9687 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9688   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9689 
 9690   ins_cost(INSN_COST * 2);
 9691   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9692 
 9693   ins_encode %{
 9694     __ csel(as_Register($dst$$reg),
 9695             zr,
 9696             as_Register($src$$reg),
 9697             (Assembler::Condition)$cmp$$cmpcode);
 9698   %}
 9699 
 9700   ins_pipe(icond_reg);
 9701 %}
 9702 
 9703 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9704   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9705 
 9706   ins_cost(INSN_COST * 2);
 9707   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9708 
 9709   ins_encode %{
 9710     __ csel(as_Register($dst$$reg),
 9711             as_Register($src$$reg),
 9712             zr,
 9713             (Assembler::Condition)$cmp$$cmpcode);
 9714   %}
 9715 
 9716   ins_pipe(icond_reg);
 9717 %}
 9718 
 9719 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9720   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9721 
 9722   ins_cost(INSN_COST * 2);
 9723   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9724 
 9725   ins_encode %{
 9726     __ csel(as_Register($dst$$reg),
 9727             as_Register($src$$reg),
 9728             zr,
 9729             (Assembler::Condition)$cmp$$cmpcode);
 9730   %}
 9731 
 9732   ins_pipe(icond_reg);
 9733 %}
 9734 
 9735 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9736   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9737 
 9738   ins_cost(INSN_COST * 2);
 9739   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9740 
 9741   ins_encode %{
 9742     __ cselw(as_Register($dst$$reg),
 9743              as_Register($src2$$reg),
 9744              as_Register($src1$$reg),
 9745              (Assembler::Condition)$cmp$$cmpcode);
 9746   %}
 9747 
 9748   ins_pipe(icond_reg_reg);
 9749 %}
 9750 
 9751 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9752   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9753 
 9754   ins_cost(INSN_COST * 2);
 9755   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9756 
 9757   ins_encode %{
 9758     __ cselw(as_Register($dst$$reg),
 9759              as_Register($src2$$reg),
 9760              as_Register($src1$$reg),
 9761              (Assembler::Condition)$cmp$$cmpcode);
 9762   %}
 9763 
 9764   ins_pipe(icond_reg_reg);
 9765 %}
 9766 
 9767 // special cases where one arg is zero
 9768 
 9769 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9770   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9771 
 9772   ins_cost(INSN_COST * 2);
 9773   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9774 
 9775   ins_encode %{
 9776     __ cselw(as_Register($dst$$reg),
 9777              zr,
 9778              as_Register($src$$reg),
 9779              (Assembler::Condition)$cmp$$cmpcode);
 9780   %}
 9781 
 9782   ins_pipe(icond_reg);
 9783 %}
 9784 
 9785 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9786   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9787 
 9788   ins_cost(INSN_COST * 2);
 9789   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9790 
 9791   ins_encode %{
 9792     __ cselw(as_Register($dst$$reg),
 9793              zr,
 9794              as_Register($src$$reg),
 9795              (Assembler::Condition)$cmp$$cmpcode);
 9796   %}
 9797 
 9798   ins_pipe(icond_reg);
 9799 %}
 9800 
 9801 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9802   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9803 
 9804   ins_cost(INSN_COST * 2);
 9805   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9806 
 9807   ins_encode %{
 9808     __ cselw(as_Register($dst$$reg),
 9809              as_Register($src$$reg),
 9810              zr,
 9811              (Assembler::Condition)$cmp$$cmpcode);
 9812   %}
 9813 
 9814   ins_pipe(icond_reg);
 9815 %}
 9816 
 9817 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9818   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9819 
 9820   ins_cost(INSN_COST * 2);
 9821   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9822 
 9823   ins_encode %{
 9824     __ cselw(as_Register($dst$$reg),
 9825              as_Register($src$$reg),
 9826              zr,
 9827              (Assembler::Condition)$cmp$$cmpcode);
 9828   %}
 9829 
 9830   ins_pipe(icond_reg);
 9831 %}
 9832 
 9833 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9834 %{
 9835   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9836 
 9837   ins_cost(INSN_COST * 3);
 9838 
 9839   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9840   ins_encode %{
 9841     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9842     __ fcsels(as_FloatRegister($dst$$reg),
 9843               as_FloatRegister($src2$$reg),
 9844               as_FloatRegister($src1$$reg),
 9845               cond);
 9846   %}
 9847 
 9848   ins_pipe(fp_cond_reg_reg_s);
 9849 %}
 9850 
 9851 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
 9852 %{
 9853   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9854 
 9855   ins_cost(INSN_COST * 3);
 9856 
 9857   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9858   ins_encode %{
 9859     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9860     __ fcsels(as_FloatRegister($dst$$reg),
 9861               as_FloatRegister($src2$$reg),
 9862               as_FloatRegister($src1$$reg),
 9863               cond);
 9864   %}
 9865 
 9866   ins_pipe(fp_cond_reg_reg_s);
 9867 %}
 9868 
 9869 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
 9870 %{
 9871   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9872 
 9873   ins_cost(INSN_COST * 3);
 9874 
 9875   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9876   ins_encode %{
 9877     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9878     __ fcseld(as_FloatRegister($dst$$reg),
 9879               as_FloatRegister($src2$$reg),
 9880               as_FloatRegister($src1$$reg),
 9881               cond);
 9882   %}
 9883 
 9884   ins_pipe(fp_cond_reg_reg_d);
 9885 %}
 9886 
 9887 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
 9888 %{
 9889   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9890 
 9891   ins_cost(INSN_COST * 3);
 9892 
 9893   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9894   ins_encode %{
 9895     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9896     __ fcseld(as_FloatRegister($dst$$reg),
 9897               as_FloatRegister($src2$$reg),
 9898               as_FloatRegister($src1$$reg),
 9899               cond);
 9900   %}
 9901 
 9902   ins_pipe(fp_cond_reg_reg_d);
 9903 %}
 9904 
 9905 // ============================================================================
 9906 // Arithmetic Instructions
 9907 //
 9908 
 9909 // Integer Addition
 9910 
 9911 // TODO
 9912 // these currently employ operations which do not set CR and hence are
 9913 // not flagged as killing CR but we would like to isolate the cases
 9914 // where we want to set flags from those where we don't. need to work
 9915 // out how to do that.
 9916 
 9917 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9918   match(Set dst (AddI src1 src2));
 9919 
 9920   ins_cost(INSN_COST);
 9921   format %{ "addw  $dst, $src1, $src2" %}
 9922 
 9923   ins_encode %{
 9924     __ addw(as_Register($dst$$reg),
 9925             as_Register($src1$$reg),
 9926             as_Register($src2$$reg));
 9927   %}
 9928 
 9929   ins_pipe(ialu_reg_reg);
 9930 %}
 9931 
 9932 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9933   match(Set dst (AddI src1 src2));
 9934 
 9935   ins_cost(INSN_COST);
 9936   format %{ "addw $dst, $src1, $src2" %}
 9937 
 9938   // use opcode to indicate that this is an add not a sub
 9939   opcode(0x0);
 9940 
 9941   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9942 
 9943   ins_pipe(ialu_reg_imm);
 9944 %}
 9945 
 9946 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
 9947   match(Set dst (AddI (ConvL2I src1) src2));
 9948 
 9949   ins_cost(INSN_COST);
 9950   format %{ "addw $dst, $src1, $src2" %}
 9951 
 9952   // use opcode to indicate that this is an add not a sub
 9953   opcode(0x0);
 9954 
 9955   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9956 
 9957   ins_pipe(ialu_reg_imm);
 9958 %}
 9959 
 9960 // Pointer Addition
 9961 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
 9962   match(Set dst (AddP src1 src2));
 9963 
 9964   ins_cost(INSN_COST);
 9965   format %{ "add $dst, $src1, $src2\t# ptr" %}
 9966 
 9967   ins_encode %{
 9968     __ add(as_Register($dst$$reg),
 9969            as_Register($src1$$reg),
 9970            as_Register($src2$$reg));
 9971   %}
 9972 
 9973   ins_pipe(ialu_reg_reg);
 9974 %}
 9975 
 9976 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
 9977   match(Set dst (AddP src1 (ConvI2L src2)));
 9978 
 9979   ins_cost(1.9 * INSN_COST);
 9980   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
 9981 
 9982   ins_encode %{
 9983     __ add(as_Register($dst$$reg),
 9984            as_Register($src1$$reg),
 9985            as_Register($src2$$reg), ext::sxtw);
 9986   %}
 9987 
 9988   ins_pipe(ialu_reg_reg);
 9989 %}
 9990 
 9991 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
 9992   match(Set dst (AddP src1 (LShiftL src2 scale)));
 9993 
 9994   ins_cost(1.9 * INSN_COST);
 9995   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
 9996 
 9997   ins_encode %{
 9998     __ lea(as_Register($dst$$reg),
 9999            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10000                    Address::lsl($scale$$constant)));
10001   %}
10002 
10003   ins_pipe(ialu_reg_reg_shift);
10004 %}
10005 
10006 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10007   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10008 
10009   ins_cost(1.9 * INSN_COST);
10010   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10011 
10012   ins_encode %{
10013     __ lea(as_Register($dst$$reg),
10014            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10015                    Address::sxtw($scale$$constant)));
10016   %}
10017 
10018   ins_pipe(ialu_reg_reg_shift);
10019 %}
10020 
10021 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10022   match(Set dst (LShiftL (ConvI2L src) scale));
10023 
10024   ins_cost(INSN_COST);
10025   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10026 
10027   ins_encode %{
10028     __ sbfiz(as_Register($dst$$reg),
10029           as_Register($src$$reg),
10030           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10031   %}
10032 
10033   ins_pipe(ialu_reg_shift);
10034 %}
10035 
10036 // Pointer Immediate Addition
10037 // n.b. this needs to be more expensive than using an indirect memory
10038 // operand
10039 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10040   match(Set dst (AddP src1 src2));
10041 
10042   ins_cost(INSN_COST);
10043   format %{ "add $dst, $src1, $src2\t# ptr" %}
10044 
10045   // use opcode to indicate that this is an add not a sub
10046   opcode(0x0);
10047 
10048   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10049 
10050   ins_pipe(ialu_reg_imm);
10051 %}
10052 
10053 // Long Addition
10054 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10055 
10056   match(Set dst (AddL src1 src2));
10057 
10058   ins_cost(INSN_COST);
10059   format %{ "add  $dst, $src1, $src2" %}
10060 
10061   ins_encode %{
10062     __ add(as_Register($dst$$reg),
10063            as_Register($src1$$reg),
10064            as_Register($src2$$reg));
10065   %}
10066 
10067   ins_pipe(ialu_reg_reg);
10068 %}
10069 
10070 // No constant pool entries requiredLong Immediate Addition.
10071 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10072   match(Set dst (AddL src1 src2));
10073 
10074   ins_cost(INSN_COST);
10075   format %{ "add $dst, $src1, $src2" %}
10076 
10077   // use opcode to indicate that this is an add not a sub
10078   opcode(0x0);
10079 
10080   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10081 
10082   ins_pipe(ialu_reg_imm);
10083 %}
10084 
10085 // Integer Subtraction
10086 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10087   match(Set dst (SubI src1 src2));
10088 
10089   ins_cost(INSN_COST);
10090   format %{ "subw  $dst, $src1, $src2" %}
10091 
10092   ins_encode %{
10093     __ subw(as_Register($dst$$reg),
10094             as_Register($src1$$reg),
10095             as_Register($src2$$reg));
10096   %}
10097 
10098   ins_pipe(ialu_reg_reg);
10099 %}
10100 
10101 // Immediate Subtraction
10102 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10103   match(Set dst (SubI src1 src2));
10104 
10105   ins_cost(INSN_COST);
10106   format %{ "subw $dst, $src1, $src2" %}
10107 
10108   // use opcode to indicate that this is a sub not an add
10109   opcode(0x1);
10110 
10111   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10112 
10113   ins_pipe(ialu_reg_imm);
10114 %}
10115 
10116 // Long Subtraction
10117 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10118 
10119   match(Set dst (SubL src1 src2));
10120 
10121   ins_cost(INSN_COST);
10122   format %{ "sub  $dst, $src1, $src2" %}
10123 
10124   ins_encode %{
10125     __ sub(as_Register($dst$$reg),
10126            as_Register($src1$$reg),
10127            as_Register($src2$$reg));
10128   %}
10129 
10130   ins_pipe(ialu_reg_reg);
10131 %}
10132 
10133 // No constant pool entries requiredLong Immediate Subtraction.
10134 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10135   match(Set dst (SubL src1 src2));
10136 
10137   ins_cost(INSN_COST);
10138   format %{ "sub$dst, $src1, $src2" %}
10139 
10140   // use opcode to indicate that this is a sub not an add
10141   opcode(0x1);
10142 
10143   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10144 
10145   ins_pipe(ialu_reg_imm);
10146 %}
10147 
10148 // Integer Negation (special case for sub)
10149 
10150 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10151   match(Set dst (SubI zero src));
10152 
10153   ins_cost(INSN_COST);
10154   format %{ "negw $dst, $src\t# int" %}
10155 
10156   ins_encode %{
10157     __ negw(as_Register($dst$$reg),
10158             as_Register($src$$reg));
10159   %}
10160 
10161   ins_pipe(ialu_reg);
10162 %}
10163 
10164 // Long Negation
10165 
10166 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10167   match(Set dst (SubL zero src));
10168 
10169   ins_cost(INSN_COST);
10170   format %{ "neg $dst, $src\t# long" %}
10171 
10172   ins_encode %{
10173     __ neg(as_Register($dst$$reg),
10174            as_Register($src$$reg));
10175   %}
10176 
10177   ins_pipe(ialu_reg);
10178 %}
10179 
10180 // Integer Multiply
10181 
10182 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10183   match(Set dst (MulI src1 src2));
10184 
10185   ins_cost(INSN_COST * 3);
10186   format %{ "mulw  $dst, $src1, $src2" %}
10187 
10188   ins_encode %{
10189     __ mulw(as_Register($dst$$reg),
10190             as_Register($src1$$reg),
10191             as_Register($src2$$reg));
10192   %}
10193 
10194   ins_pipe(imul_reg_reg);
10195 %}
10196 
10197 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10198   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10199 
10200   ins_cost(INSN_COST * 3);
10201   format %{ "smull  $dst, $src1, $src2" %}
10202 
10203   ins_encode %{
10204     __ smull(as_Register($dst$$reg),
10205              as_Register($src1$$reg),
10206              as_Register($src2$$reg));
10207   %}
10208 
10209   ins_pipe(imul_reg_reg);
10210 %}
10211 
10212 // Long Multiply
10213 
10214 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10215   match(Set dst (MulL src1 src2));
10216 
10217   ins_cost(INSN_COST * 5);
10218   format %{ "mul  $dst, $src1, $src2" %}
10219 
10220   ins_encode %{
10221     __ mul(as_Register($dst$$reg),
10222            as_Register($src1$$reg),
10223            as_Register($src2$$reg));
10224   %}
10225 
10226   ins_pipe(lmul_reg_reg);
10227 %}
10228 
10229 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10230 %{
10231   match(Set dst (MulHiL src1 src2));
10232 
10233   ins_cost(INSN_COST * 7);
10234   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10235 
10236   ins_encode %{
10237     __ smulh(as_Register($dst$$reg),
10238              as_Register($src1$$reg),
10239              as_Register($src2$$reg));
10240   %}
10241 
10242   ins_pipe(lmul_reg_reg);
10243 %}
10244 
10245 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10246 %{
10247   match(Set dst (UMulHiL src1 src2));
10248 
10249   ins_cost(INSN_COST * 7);
10250   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10251 
10252   ins_encode %{
10253     __ umulh(as_Register($dst$$reg),
10254              as_Register($src1$$reg),
10255              as_Register($src2$$reg));
10256   %}
10257 
10258   ins_pipe(lmul_reg_reg);
10259 %}
10260 
10261 // Combined Integer Multiply & Add/Sub
10262 
10263 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10264   match(Set dst (AddI src3 (MulI src1 src2)));
10265 
10266   ins_cost(INSN_COST * 3);
10267   format %{ "madd  $dst, $src1, $src2, $src3" %}
10268 
10269   ins_encode %{
10270     __ maddw(as_Register($dst$$reg),
10271              as_Register($src1$$reg),
10272              as_Register($src2$$reg),
10273              as_Register($src3$$reg));
10274   %}
10275 
10276   ins_pipe(imac_reg_reg);
10277 %}
10278 
10279 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10280   match(Set dst (SubI src3 (MulI src1 src2)));
10281 
10282   ins_cost(INSN_COST * 3);
10283   format %{ "msub  $dst, $src1, $src2, $src3" %}
10284 
10285   ins_encode %{
10286     __ msubw(as_Register($dst$$reg),
10287              as_Register($src1$$reg),
10288              as_Register($src2$$reg),
10289              as_Register($src3$$reg));
10290   %}
10291 
10292   ins_pipe(imac_reg_reg);
10293 %}
10294 
10295 // Combined Integer Multiply & Neg
10296 
10297 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10298   match(Set dst (MulI (SubI zero src1) src2));
10299 
10300   ins_cost(INSN_COST * 3);
10301   format %{ "mneg  $dst, $src1, $src2" %}
10302 
10303   ins_encode %{
10304     __ mnegw(as_Register($dst$$reg),
10305              as_Register($src1$$reg),
10306              as_Register($src2$$reg));
10307   %}
10308 
10309   ins_pipe(imac_reg_reg);
10310 %}
10311 
10312 // Combined Long Multiply & Add/Sub
10313 
10314 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10315   match(Set dst (AddL src3 (MulL src1 src2)));
10316 
10317   ins_cost(INSN_COST * 5);
10318   format %{ "madd  $dst, $src1, $src2, $src3" %}
10319 
10320   ins_encode %{
10321     __ madd(as_Register($dst$$reg),
10322             as_Register($src1$$reg),
10323             as_Register($src2$$reg),
10324             as_Register($src3$$reg));
10325   %}
10326 
10327   ins_pipe(lmac_reg_reg);
10328 %}
10329 
10330 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10331   match(Set dst (SubL src3 (MulL src1 src2)));
10332 
10333   ins_cost(INSN_COST * 5);
10334   format %{ "msub  $dst, $src1, $src2, $src3" %}
10335 
10336   ins_encode %{
10337     __ msub(as_Register($dst$$reg),
10338             as_Register($src1$$reg),
10339             as_Register($src2$$reg),
10340             as_Register($src3$$reg));
10341   %}
10342 
10343   ins_pipe(lmac_reg_reg);
10344 %}
10345 
10346 // Combined Long Multiply & Neg
10347 
10348 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10349   match(Set dst (MulL (SubL zero src1) src2));
10350 
10351   ins_cost(INSN_COST * 5);
10352   format %{ "mneg  $dst, $src1, $src2" %}
10353 
10354   ins_encode %{
10355     __ mneg(as_Register($dst$$reg),
10356             as_Register($src1$$reg),
10357             as_Register($src2$$reg));
10358   %}
10359 
10360   ins_pipe(lmac_reg_reg);
10361 %}
10362 
10363 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10364 
10365 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10366   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10367 
10368   ins_cost(INSN_COST * 3);
10369   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10370 
10371   ins_encode %{
10372     __ smaddl(as_Register($dst$$reg),
10373               as_Register($src1$$reg),
10374               as_Register($src2$$reg),
10375               as_Register($src3$$reg));
10376   %}
10377 
10378   ins_pipe(imac_reg_reg);
10379 %}
10380 
10381 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10382   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10383 
10384   ins_cost(INSN_COST * 3);
10385   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10386 
10387   ins_encode %{
10388     __ smsubl(as_Register($dst$$reg),
10389               as_Register($src1$$reg),
10390               as_Register($src2$$reg),
10391               as_Register($src3$$reg));
10392   %}
10393 
10394   ins_pipe(imac_reg_reg);
10395 %}
10396 
10397 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10398   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10399 
10400   ins_cost(INSN_COST * 3);
10401   format %{ "smnegl  $dst, $src1, $src2" %}
10402 
10403   ins_encode %{
10404     __ smnegl(as_Register($dst$$reg),
10405               as_Register($src1$$reg),
10406               as_Register($src2$$reg));
10407   %}
10408 
10409   ins_pipe(imac_reg_reg);
10410 %}
10411 
10412 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10413 
10414 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10415   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10416 
10417   ins_cost(INSN_COST * 5);
10418   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10419             "maddw $dst, $src3, $src4, rscratch1" %}
10420 
10421   ins_encode %{
10422     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10423     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10424 
10425   ins_pipe(imac_reg_reg);
10426 %}
10427 
10428 // Integer Divide
10429 
10430 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10431   match(Set dst (DivI src1 src2));
10432 
10433   ins_cost(INSN_COST * 19);
10434   format %{ "sdivw  $dst, $src1, $src2" %}
10435 
10436   ins_encode(aarch64_enc_divw(dst, src1, src2));
10437   ins_pipe(idiv_reg_reg);
10438 %}
10439 
10440 // Long Divide
10441 
10442 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10443   match(Set dst (DivL src1 src2));
10444 
10445   ins_cost(INSN_COST * 35);
10446   format %{ "sdiv   $dst, $src1, $src2" %}
10447 
10448   ins_encode(aarch64_enc_div(dst, src1, src2));
10449   ins_pipe(ldiv_reg_reg);
10450 %}
10451 
10452 // Integer Remainder
10453 
10454 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10455   match(Set dst (ModI src1 src2));
10456 
10457   ins_cost(INSN_COST * 22);
10458   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10459             "msubw  $dst, rscratch1, $src2, $src1" %}
10460 
10461   ins_encode(aarch64_enc_modw(dst, src1, src2));
10462   ins_pipe(idiv_reg_reg);
10463 %}
10464 
10465 // Long Remainder
10466 
10467 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10468   match(Set dst (ModL src1 src2));
10469 
10470   ins_cost(INSN_COST * 38);
10471   format %{ "sdiv   rscratch1, $src1, $src2\n"
10472             "msub   $dst, rscratch1, $src2, $src1" %}
10473 
10474   ins_encode(aarch64_enc_mod(dst, src1, src2));
10475   ins_pipe(ldiv_reg_reg);
10476 %}
10477 
10478 // Unsigned Integer Divide
10479 
10480 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10481   match(Set dst (UDivI src1 src2));
10482 
10483   ins_cost(INSN_COST * 19);
10484   format %{ "udivw  $dst, $src1, $src2" %}
10485 
10486   ins_encode %{
10487     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10488   %}
10489 
10490   ins_pipe(idiv_reg_reg);
10491 %}
10492 
10493 //  Unsigned Long Divide
10494 
10495 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10496   match(Set dst (UDivL src1 src2));
10497 
10498   ins_cost(INSN_COST * 35);
10499   format %{ "udiv   $dst, $src1, $src2" %}
10500 
10501   ins_encode %{
10502     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10503   %}
10504 
10505   ins_pipe(ldiv_reg_reg);
10506 %}
10507 
10508 // Unsigned Integer Remainder
10509 
10510 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10511   match(Set dst (UModI src1 src2));
10512 
10513   ins_cost(INSN_COST * 22);
10514   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10515             "msubw  $dst, rscratch1, $src2, $src1" %}
10516 
10517   ins_encode %{
10518     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10519     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10520   %}
10521 
10522   ins_pipe(idiv_reg_reg);
10523 %}
10524 
10525 // Unsigned Long Remainder
10526 
10527 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10528   match(Set dst (UModL src1 src2));
10529 
10530   ins_cost(INSN_COST * 38);
10531   format %{ "udiv   rscratch1, $src1, $src2\n"
10532             "msub   $dst, rscratch1, $src2, $src1" %}
10533 
10534   ins_encode %{
10535     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10536     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10537   %}
10538 
10539   ins_pipe(ldiv_reg_reg);
10540 %}
10541 
10542 // Integer Shifts
10543 
10544 // Shift Left Register
10545 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10546   match(Set dst (LShiftI src1 src2));
10547 
10548   ins_cost(INSN_COST * 2);
10549   format %{ "lslvw  $dst, $src1, $src2" %}
10550 
10551   ins_encode %{
10552     __ lslvw(as_Register($dst$$reg),
10553              as_Register($src1$$reg),
10554              as_Register($src2$$reg));
10555   %}
10556 
10557   ins_pipe(ialu_reg_reg_vshift);
10558 %}
10559 
10560 // Shift Left Immediate
10561 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10562   match(Set dst (LShiftI src1 src2));
10563 
10564   ins_cost(INSN_COST);
10565   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10566 
10567   ins_encode %{
10568     __ lslw(as_Register($dst$$reg),
10569             as_Register($src1$$reg),
10570             $src2$$constant & 0x1f);
10571   %}
10572 
10573   ins_pipe(ialu_reg_shift);
10574 %}
10575 
10576 // Shift Right Logical Register
10577 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10578   match(Set dst (URShiftI src1 src2));
10579 
10580   ins_cost(INSN_COST * 2);
10581   format %{ "lsrvw  $dst, $src1, $src2" %}
10582 
10583   ins_encode %{
10584     __ lsrvw(as_Register($dst$$reg),
10585              as_Register($src1$$reg),
10586              as_Register($src2$$reg));
10587   %}
10588 
10589   ins_pipe(ialu_reg_reg_vshift);
10590 %}
10591 
10592 // Shift Right Logical Immediate
10593 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10594   match(Set dst (URShiftI src1 src2));
10595 
10596   ins_cost(INSN_COST);
10597   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10598 
10599   ins_encode %{
10600     __ lsrw(as_Register($dst$$reg),
10601             as_Register($src1$$reg),
10602             $src2$$constant & 0x1f);
10603   %}
10604 
10605   ins_pipe(ialu_reg_shift);
10606 %}
10607 
10608 // Shift Right Arithmetic Register
10609 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10610   match(Set dst (RShiftI src1 src2));
10611 
10612   ins_cost(INSN_COST * 2);
10613   format %{ "asrvw  $dst, $src1, $src2" %}
10614 
10615   ins_encode %{
10616     __ asrvw(as_Register($dst$$reg),
10617              as_Register($src1$$reg),
10618              as_Register($src2$$reg));
10619   %}
10620 
10621   ins_pipe(ialu_reg_reg_vshift);
10622 %}
10623 
10624 // Shift Right Arithmetic Immediate
10625 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10626   match(Set dst (RShiftI src1 src2));
10627 
10628   ins_cost(INSN_COST);
10629   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10630 
10631   ins_encode %{
10632     __ asrw(as_Register($dst$$reg),
10633             as_Register($src1$$reg),
10634             $src2$$constant & 0x1f);
10635   %}
10636 
10637   ins_pipe(ialu_reg_shift);
10638 %}
10639 
10640 // Combined Int Mask and Right Shift (using UBFM)
10641 // TODO
10642 
10643 // Long Shifts
10644 
10645 // Shift Left Register
10646 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10647   match(Set dst (LShiftL src1 src2));
10648 
10649   ins_cost(INSN_COST * 2);
10650   format %{ "lslv  $dst, $src1, $src2" %}
10651 
10652   ins_encode %{
10653     __ lslv(as_Register($dst$$reg),
10654             as_Register($src1$$reg),
10655             as_Register($src2$$reg));
10656   %}
10657 
10658   ins_pipe(ialu_reg_reg_vshift);
10659 %}
10660 
10661 // Shift Left Immediate
10662 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10663   match(Set dst (LShiftL src1 src2));
10664 
10665   ins_cost(INSN_COST);
10666   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10667 
10668   ins_encode %{
10669     __ lsl(as_Register($dst$$reg),
10670             as_Register($src1$$reg),
10671             $src2$$constant & 0x3f);
10672   %}
10673 
10674   ins_pipe(ialu_reg_shift);
10675 %}
10676 
10677 // Shift Right Logical Register
10678 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10679   match(Set dst (URShiftL src1 src2));
10680 
10681   ins_cost(INSN_COST * 2);
10682   format %{ "lsrv  $dst, $src1, $src2" %}
10683 
10684   ins_encode %{
10685     __ lsrv(as_Register($dst$$reg),
10686             as_Register($src1$$reg),
10687             as_Register($src2$$reg));
10688   %}
10689 
10690   ins_pipe(ialu_reg_reg_vshift);
10691 %}
10692 
10693 // Shift Right Logical Immediate
10694 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10695   match(Set dst (URShiftL src1 src2));
10696 
10697   ins_cost(INSN_COST);
10698   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10699 
10700   ins_encode %{
10701     __ lsr(as_Register($dst$$reg),
10702            as_Register($src1$$reg),
10703            $src2$$constant & 0x3f);
10704   %}
10705 
10706   ins_pipe(ialu_reg_shift);
10707 %}
10708 
10709 // A special-case pattern for card table stores.
10710 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10711   match(Set dst (URShiftL (CastP2X src1) src2));
10712 
10713   ins_cost(INSN_COST);
10714   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10715 
10716   ins_encode %{
10717     __ lsr(as_Register($dst$$reg),
10718            as_Register($src1$$reg),
10719            $src2$$constant & 0x3f);
10720   %}
10721 
10722   ins_pipe(ialu_reg_shift);
10723 %}
10724 
10725 // Shift Right Arithmetic Register
10726 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10727   match(Set dst (RShiftL src1 src2));
10728 
10729   ins_cost(INSN_COST * 2);
10730   format %{ "asrv  $dst, $src1, $src2" %}
10731 
10732   ins_encode %{
10733     __ asrv(as_Register($dst$$reg),
10734             as_Register($src1$$reg),
10735             as_Register($src2$$reg));
10736   %}
10737 
10738   ins_pipe(ialu_reg_reg_vshift);
10739 %}
10740 
10741 // Shift Right Arithmetic Immediate
10742 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10743   match(Set dst (RShiftL src1 src2));
10744 
10745   ins_cost(INSN_COST);
10746   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10747 
10748   ins_encode %{
10749     __ asr(as_Register($dst$$reg),
10750            as_Register($src1$$reg),
10751            $src2$$constant & 0x3f);
10752   %}
10753 
10754   ins_pipe(ialu_reg_shift);
10755 %}
10756 
10757 // BEGIN This section of the file is automatically generated. Do not edit --------------
10758 // This section is generated from aarch64_ad.m4
10759 
10760 // This pattern is automatically generated from aarch64_ad.m4
10761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10762 instruct regL_not_reg(iRegLNoSp dst,
10763                          iRegL src1, immL_M1 m1,
10764                          rFlagsReg cr) %{
10765   match(Set dst (XorL src1 m1));
10766   ins_cost(INSN_COST);
10767   format %{ "eon  $dst, $src1, zr" %}
10768 
10769   ins_encode %{
10770     __ eon(as_Register($dst$$reg),
10771               as_Register($src1$$reg),
10772               zr,
10773               Assembler::LSL, 0);
10774   %}
10775 
10776   ins_pipe(ialu_reg);
10777 %}
10778 
10779 // This pattern is automatically generated from aarch64_ad.m4
10780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10781 instruct regI_not_reg(iRegINoSp dst,
10782                          iRegIorL2I src1, immI_M1 m1,
10783                          rFlagsReg cr) %{
10784   match(Set dst (XorI src1 m1));
10785   ins_cost(INSN_COST);
10786   format %{ "eonw  $dst, $src1, zr" %}
10787 
10788   ins_encode %{
10789     __ eonw(as_Register($dst$$reg),
10790               as_Register($src1$$reg),
10791               zr,
10792               Assembler::LSL, 0);
10793   %}
10794 
10795   ins_pipe(ialu_reg);
10796 %}
10797 
10798 // This pattern is automatically generated from aarch64_ad.m4
10799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10800 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10801                               immI0 zero, iRegIorL2I src1, immI src2) %{
10802   match(Set dst (SubI zero (URShiftI src1 src2)));
10803 
10804   ins_cost(1.9 * INSN_COST);
10805   format %{ "negw  $dst, $src1, LSR $src2" %}
10806 
10807   ins_encode %{
10808     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10809             Assembler::LSR, $src2$$constant & 0x1f);
10810   %}
10811 
10812   ins_pipe(ialu_reg_shift);
10813 %}
10814 
10815 // This pattern is automatically generated from aarch64_ad.m4
10816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10817 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10818                               immI0 zero, iRegIorL2I src1, immI src2) %{
10819   match(Set dst (SubI zero (RShiftI src1 src2)));
10820 
10821   ins_cost(1.9 * INSN_COST);
10822   format %{ "negw  $dst, $src1, ASR $src2" %}
10823 
10824   ins_encode %{
10825     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10826             Assembler::ASR, $src2$$constant & 0x1f);
10827   %}
10828 
10829   ins_pipe(ialu_reg_shift);
10830 %}
10831 
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10835                               immI0 zero, iRegIorL2I src1, immI src2) %{
10836   match(Set dst (SubI zero (LShiftI src1 src2)));
10837 
10838   ins_cost(1.9 * INSN_COST);
10839   format %{ "negw  $dst, $src1, LSL $src2" %}
10840 
10841   ins_encode %{
10842     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10843             Assembler::LSL, $src2$$constant & 0x1f);
10844   %}
10845 
10846   ins_pipe(ialu_reg_shift);
10847 %}
10848 
10849 // This pattern is automatically generated from aarch64_ad.m4
10850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10851 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10852                               immL0 zero, iRegL src1, immI src2) %{
10853   match(Set dst (SubL zero (URShiftL src1 src2)));
10854 
10855   ins_cost(1.9 * INSN_COST);
10856   format %{ "neg  $dst, $src1, LSR $src2" %}
10857 
10858   ins_encode %{
10859     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10860             Assembler::LSR, $src2$$constant & 0x3f);
10861   %}
10862 
10863   ins_pipe(ialu_reg_shift);
10864 %}
10865 
10866 // This pattern is automatically generated from aarch64_ad.m4
10867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10868 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10869                               immL0 zero, iRegL src1, immI src2) %{
10870   match(Set dst (SubL zero (RShiftL src1 src2)));
10871 
10872   ins_cost(1.9 * INSN_COST);
10873   format %{ "neg  $dst, $src1, ASR $src2" %}
10874 
10875   ins_encode %{
10876     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10877             Assembler::ASR, $src2$$constant & 0x3f);
10878   %}
10879 
10880   ins_pipe(ialu_reg_shift);
10881 %}
10882 
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10886                               immL0 zero, iRegL src1, immI src2) %{
10887   match(Set dst (SubL zero (LShiftL src1 src2)));
10888 
10889   ins_cost(1.9 * INSN_COST);
10890   format %{ "neg  $dst, $src1, LSL $src2" %}
10891 
10892   ins_encode %{
10893     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10894             Assembler::LSL, $src2$$constant & 0x3f);
10895   %}
10896 
10897   ins_pipe(ialu_reg_shift);
10898 %}
10899 
10900 // This pattern is automatically generated from aarch64_ad.m4
10901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10902 instruct AndI_reg_not_reg(iRegINoSp dst,
10903                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10904   match(Set dst (AndI src1 (XorI src2 m1)));
10905   ins_cost(INSN_COST);
10906   format %{ "bicw  $dst, $src1, $src2" %}
10907 
10908   ins_encode %{
10909     __ bicw(as_Register($dst$$reg),
10910               as_Register($src1$$reg),
10911               as_Register($src2$$reg),
10912               Assembler::LSL, 0);
10913   %}
10914 
10915   ins_pipe(ialu_reg_reg);
10916 %}
10917 
10918 // This pattern is automatically generated from aarch64_ad.m4
10919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10920 instruct AndL_reg_not_reg(iRegLNoSp dst,
10921                          iRegL src1, iRegL src2, immL_M1 m1) %{
10922   match(Set dst (AndL src1 (XorL src2 m1)));
10923   ins_cost(INSN_COST);
10924   format %{ "bic  $dst, $src1, $src2" %}
10925 
10926   ins_encode %{
10927     __ bic(as_Register($dst$$reg),
10928               as_Register($src1$$reg),
10929               as_Register($src2$$reg),
10930               Assembler::LSL, 0);
10931   %}
10932 
10933   ins_pipe(ialu_reg_reg);
10934 %}
10935 
10936 // This pattern is automatically generated from aarch64_ad.m4
10937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10938 instruct OrI_reg_not_reg(iRegINoSp dst,
10939                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10940   match(Set dst (OrI src1 (XorI src2 m1)));
10941   ins_cost(INSN_COST);
10942   format %{ "ornw  $dst, $src1, $src2" %}
10943 
10944   ins_encode %{
10945     __ ornw(as_Register($dst$$reg),
10946               as_Register($src1$$reg),
10947               as_Register($src2$$reg),
10948               Assembler::LSL, 0);
10949   %}
10950 
10951   ins_pipe(ialu_reg_reg);
10952 %}
10953 
10954 // This pattern is automatically generated from aarch64_ad.m4
10955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10956 instruct OrL_reg_not_reg(iRegLNoSp dst,
10957                          iRegL src1, iRegL src2, immL_M1 m1) %{
10958   match(Set dst (OrL src1 (XorL src2 m1)));
10959   ins_cost(INSN_COST);
10960   format %{ "orn  $dst, $src1, $src2" %}
10961 
10962   ins_encode %{
10963     __ orn(as_Register($dst$$reg),
10964               as_Register($src1$$reg),
10965               as_Register($src2$$reg),
10966               Assembler::LSL, 0);
10967   %}
10968 
10969   ins_pipe(ialu_reg_reg);
10970 %}
10971 
10972 // This pattern is automatically generated from aarch64_ad.m4
10973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10974 instruct XorI_reg_not_reg(iRegINoSp dst,
10975                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10976   match(Set dst (XorI m1 (XorI src2 src1)));
10977   ins_cost(INSN_COST);
10978   format %{ "eonw  $dst, $src1, $src2" %}
10979 
10980   ins_encode %{
10981     __ eonw(as_Register($dst$$reg),
10982               as_Register($src1$$reg),
10983               as_Register($src2$$reg),
10984               Assembler::LSL, 0);
10985   %}
10986 
10987   ins_pipe(ialu_reg_reg);
10988 %}
10989 
10990 // This pattern is automatically generated from aarch64_ad.m4
10991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10992 instruct XorL_reg_not_reg(iRegLNoSp dst,
10993                          iRegL src1, iRegL src2, immL_M1 m1) %{
10994   match(Set dst (XorL m1 (XorL src2 src1)));
10995   ins_cost(INSN_COST);
10996   format %{ "eon  $dst, $src1, $src2" %}
10997 
10998   ins_encode %{
10999     __ eon(as_Register($dst$$reg),
11000               as_Register($src1$$reg),
11001               as_Register($src2$$reg),
11002               Assembler::LSL, 0);
11003   %}
11004 
11005   ins_pipe(ialu_reg_reg);
11006 %}
11007 
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 // val & (-1 ^ (val >>> shift)) ==> bicw
11011 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11012                          iRegIorL2I src1, iRegIorL2I src2,
11013                          immI src3, immI_M1 src4) %{
11014   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11015   ins_cost(1.9 * INSN_COST);
11016   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11017 
11018   ins_encode %{
11019     __ bicw(as_Register($dst$$reg),
11020               as_Register($src1$$reg),
11021               as_Register($src2$$reg),
11022               Assembler::LSR,
11023               $src3$$constant & 0x1f);
11024   %}
11025 
11026   ins_pipe(ialu_reg_reg_shift);
11027 %}
11028 
11029 // This pattern is automatically generated from aarch64_ad.m4
11030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11031 // val & (-1 ^ (val >>> shift)) ==> bic
11032 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11033                          iRegL src1, iRegL src2,
11034                          immI src3, immL_M1 src4) %{
11035   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11036   ins_cost(1.9 * INSN_COST);
11037   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11038 
11039   ins_encode %{
11040     __ bic(as_Register($dst$$reg),
11041               as_Register($src1$$reg),
11042               as_Register($src2$$reg),
11043               Assembler::LSR,
11044               $src3$$constant & 0x3f);
11045   %}
11046 
11047   ins_pipe(ialu_reg_reg_shift);
11048 %}
11049 
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 // val & (-1 ^ (val >> shift)) ==> bicw
11053 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11054                          iRegIorL2I src1, iRegIorL2I src2,
11055                          immI src3, immI_M1 src4) %{
11056   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11057   ins_cost(1.9 * INSN_COST);
11058   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11059 
11060   ins_encode %{
11061     __ bicw(as_Register($dst$$reg),
11062               as_Register($src1$$reg),
11063               as_Register($src2$$reg),
11064               Assembler::ASR,
11065               $src3$$constant & 0x1f);
11066   %}
11067 
11068   ins_pipe(ialu_reg_reg_shift);
11069 %}
11070 
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 // val & (-1 ^ (val >> shift)) ==> bic
11074 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11075                          iRegL src1, iRegL src2,
11076                          immI src3, immL_M1 src4) %{
11077   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11078   ins_cost(1.9 * INSN_COST);
11079   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11080 
11081   ins_encode %{
11082     __ bic(as_Register($dst$$reg),
11083               as_Register($src1$$reg),
11084               as_Register($src2$$reg),
11085               Assembler::ASR,
11086               $src3$$constant & 0x3f);
11087   %}
11088 
11089   ins_pipe(ialu_reg_reg_shift);
11090 %}
11091 
11092 // This pattern is automatically generated from aarch64_ad.m4
11093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11094 // val & (-1 ^ (val ror shift)) ==> bicw
11095 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11096                          iRegIorL2I src1, iRegIorL2I src2,
11097                          immI src3, immI_M1 src4) %{
11098   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11099   ins_cost(1.9 * INSN_COST);
11100   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11101 
11102   ins_encode %{
11103     __ bicw(as_Register($dst$$reg),
11104               as_Register($src1$$reg),
11105               as_Register($src2$$reg),
11106               Assembler::ROR,
11107               $src3$$constant & 0x1f);
11108   %}
11109 
11110   ins_pipe(ialu_reg_reg_shift);
11111 %}
11112 
11113 // This pattern is automatically generated from aarch64_ad.m4
11114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11115 // val & (-1 ^ (val ror shift)) ==> bic
11116 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11117                          iRegL src1, iRegL src2,
11118                          immI src3, immL_M1 src4) %{
11119   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11120   ins_cost(1.9 * INSN_COST);
11121   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11122 
11123   ins_encode %{
11124     __ bic(as_Register($dst$$reg),
11125               as_Register($src1$$reg),
11126               as_Register($src2$$reg),
11127               Assembler::ROR,
11128               $src3$$constant & 0x3f);
11129   %}
11130 
11131   ins_pipe(ialu_reg_reg_shift);
11132 %}
11133 
11134 // This pattern is automatically generated from aarch64_ad.m4
11135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11136 // val & (-1 ^ (val << shift)) ==> bicw
11137 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11138                          iRegIorL2I src1, iRegIorL2I src2,
11139                          immI src3, immI_M1 src4) %{
11140   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11141   ins_cost(1.9 * INSN_COST);
11142   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11143 
11144   ins_encode %{
11145     __ bicw(as_Register($dst$$reg),
11146               as_Register($src1$$reg),
11147               as_Register($src2$$reg),
11148               Assembler::LSL,
11149               $src3$$constant & 0x1f);
11150   %}
11151 
11152   ins_pipe(ialu_reg_reg_shift);
11153 %}
11154 
11155 // This pattern is automatically generated from aarch64_ad.m4
11156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11157 // val & (-1 ^ (val << shift)) ==> bic
11158 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11159                          iRegL src1, iRegL src2,
11160                          immI src3, immL_M1 src4) %{
11161   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11162   ins_cost(1.9 * INSN_COST);
11163   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11164 
11165   ins_encode %{
11166     __ bic(as_Register($dst$$reg),
11167               as_Register($src1$$reg),
11168               as_Register($src2$$reg),
11169               Assembler::LSL,
11170               $src3$$constant & 0x3f);
11171   %}
11172 
11173   ins_pipe(ialu_reg_reg_shift);
11174 %}
11175 
11176 // This pattern is automatically generated from aarch64_ad.m4
11177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11178 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11179 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11180                          iRegIorL2I src1, iRegIorL2I src2,
11181                          immI src3, immI_M1 src4) %{
11182   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11183   ins_cost(1.9 * INSN_COST);
11184   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11185 
11186   ins_encode %{
11187     __ eonw(as_Register($dst$$reg),
11188               as_Register($src1$$reg),
11189               as_Register($src2$$reg),
11190               Assembler::LSR,
11191               $src3$$constant & 0x1f);
11192   %}
11193 
11194   ins_pipe(ialu_reg_reg_shift);
11195 %}
11196 
11197 // This pattern is automatically generated from aarch64_ad.m4
11198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11199 // val ^ (-1 ^ (val >>> shift)) ==> eon
11200 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11201                          iRegL src1, iRegL src2,
11202                          immI src3, immL_M1 src4) %{
11203   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11204   ins_cost(1.9 * INSN_COST);
11205   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11206 
11207   ins_encode %{
11208     __ eon(as_Register($dst$$reg),
11209               as_Register($src1$$reg),
11210               as_Register($src2$$reg),
11211               Assembler::LSR,
11212               $src3$$constant & 0x3f);
11213   %}
11214 
11215   ins_pipe(ialu_reg_reg_shift);
11216 %}
11217 
11218 // This pattern is automatically generated from aarch64_ad.m4
11219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11220 // val ^ (-1 ^ (val >> shift)) ==> eonw
11221 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11222                          iRegIorL2I src1, iRegIorL2I src2,
11223                          immI src3, immI_M1 src4) %{
11224   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11225   ins_cost(1.9 * INSN_COST);
11226   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11227 
11228   ins_encode %{
11229     __ eonw(as_Register($dst$$reg),
11230               as_Register($src1$$reg),
11231               as_Register($src2$$reg),
11232               Assembler::ASR,
11233               $src3$$constant & 0x1f);
11234   %}
11235 
11236   ins_pipe(ialu_reg_reg_shift);
11237 %}
11238 
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 // val ^ (-1 ^ (val >> shift)) ==> eon
11242 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11243                          iRegL src1, iRegL src2,
11244                          immI src3, immL_M1 src4) %{
11245   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11246   ins_cost(1.9 * INSN_COST);
11247   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11248 
11249   ins_encode %{
11250     __ eon(as_Register($dst$$reg),
11251               as_Register($src1$$reg),
11252               as_Register($src2$$reg),
11253               Assembler::ASR,
11254               $src3$$constant & 0x3f);
11255   %}
11256 
11257   ins_pipe(ialu_reg_reg_shift);
11258 %}
11259 
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 // val ^ (-1 ^ (val ror shift)) ==> eonw
11263 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11264                          iRegIorL2I src1, iRegIorL2I src2,
11265                          immI src3, immI_M1 src4) %{
11266   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11267   ins_cost(1.9 * INSN_COST);
11268   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11269 
11270   ins_encode %{
11271     __ eonw(as_Register($dst$$reg),
11272               as_Register($src1$$reg),
11273               as_Register($src2$$reg),
11274               Assembler::ROR,
11275               $src3$$constant & 0x1f);
11276   %}
11277 
11278   ins_pipe(ialu_reg_reg_shift);
11279 %}
11280 
11281 // This pattern is automatically generated from aarch64_ad.m4
11282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11283 // val ^ (-1 ^ (val ror shift)) ==> eon
11284 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11285                          iRegL src1, iRegL src2,
11286                          immI src3, immL_M1 src4) %{
11287   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11288   ins_cost(1.9 * INSN_COST);
11289   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11290 
11291   ins_encode %{
11292     __ eon(as_Register($dst$$reg),
11293               as_Register($src1$$reg),
11294               as_Register($src2$$reg),
11295               Assembler::ROR,
11296               $src3$$constant & 0x3f);
11297   %}
11298 
11299   ins_pipe(ialu_reg_reg_shift);
11300 %}
11301 
11302 // This pattern is automatically generated from aarch64_ad.m4
11303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11304 // val ^ (-1 ^ (val << shift)) ==> eonw
11305 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11306                          iRegIorL2I src1, iRegIorL2I src2,
11307                          immI src3, immI_M1 src4) %{
11308   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11309   ins_cost(1.9 * INSN_COST);
11310   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11311 
11312   ins_encode %{
11313     __ eonw(as_Register($dst$$reg),
11314               as_Register($src1$$reg),
11315               as_Register($src2$$reg),
11316               Assembler::LSL,
11317               $src3$$constant & 0x1f);
11318   %}
11319 
11320   ins_pipe(ialu_reg_reg_shift);
11321 %}
11322 
11323 // This pattern is automatically generated from aarch64_ad.m4
11324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11325 // val ^ (-1 ^ (val << shift)) ==> eon
11326 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11327                          iRegL src1, iRegL src2,
11328                          immI src3, immL_M1 src4) %{
11329   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11330   ins_cost(1.9 * INSN_COST);
11331   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11332 
11333   ins_encode %{
11334     __ eon(as_Register($dst$$reg),
11335               as_Register($src1$$reg),
11336               as_Register($src2$$reg),
11337               Assembler::LSL,
11338               $src3$$constant & 0x3f);
11339   %}
11340 
11341   ins_pipe(ialu_reg_reg_shift);
11342 %}
11343 
11344 // This pattern is automatically generated from aarch64_ad.m4
11345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11346 // val | (-1 ^ (val >>> shift)) ==> ornw
11347 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11348                          iRegIorL2I src1, iRegIorL2I src2,
11349                          immI src3, immI_M1 src4) %{
11350   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11351   ins_cost(1.9 * INSN_COST);
11352   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11353 
11354   ins_encode %{
11355     __ ornw(as_Register($dst$$reg),
11356               as_Register($src1$$reg),
11357               as_Register($src2$$reg),
11358               Assembler::LSR,
11359               $src3$$constant & 0x1f);
11360   %}
11361 
11362   ins_pipe(ialu_reg_reg_shift);
11363 %}
11364 
11365 // This pattern is automatically generated from aarch64_ad.m4
11366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11367 // val | (-1 ^ (val >>> shift)) ==> orn
11368 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11369                          iRegL src1, iRegL src2,
11370                          immI src3, immL_M1 src4) %{
11371   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11372   ins_cost(1.9 * INSN_COST);
11373   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11374 
11375   ins_encode %{
11376     __ orn(as_Register($dst$$reg),
11377               as_Register($src1$$reg),
11378               as_Register($src2$$reg),
11379               Assembler::LSR,
11380               $src3$$constant & 0x3f);
11381   %}
11382 
11383   ins_pipe(ialu_reg_reg_shift);
11384 %}
11385 
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388 // val | (-1 ^ (val >> shift)) ==> ornw
11389 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11390                          iRegIorL2I src1, iRegIorL2I src2,
11391                          immI src3, immI_M1 src4) %{
11392   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11393   ins_cost(1.9 * INSN_COST);
11394   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11395 
11396   ins_encode %{
11397     __ ornw(as_Register($dst$$reg),
11398               as_Register($src1$$reg),
11399               as_Register($src2$$reg),
11400               Assembler::ASR,
11401               $src3$$constant & 0x1f);
11402   %}
11403 
11404   ins_pipe(ialu_reg_reg_shift);
11405 %}
11406 
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409 // val | (-1 ^ (val >> shift)) ==> orn
11410 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11411                          iRegL src1, iRegL src2,
11412                          immI src3, immL_M1 src4) %{
11413   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11414   ins_cost(1.9 * INSN_COST);
11415   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11416 
11417   ins_encode %{
11418     __ orn(as_Register($dst$$reg),
11419               as_Register($src1$$reg),
11420               as_Register($src2$$reg),
11421               Assembler::ASR,
11422               $src3$$constant & 0x3f);
11423   %}
11424 
11425   ins_pipe(ialu_reg_reg_shift);
11426 %}
11427 
11428 // This pattern is automatically generated from aarch64_ad.m4
11429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11430 // val | (-1 ^ (val ror shift)) ==> ornw
11431 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11432                          iRegIorL2I src1, iRegIorL2I src2,
11433                          immI src3, immI_M1 src4) %{
11434   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11435   ins_cost(1.9 * INSN_COST);
11436   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11437 
11438   ins_encode %{
11439     __ ornw(as_Register($dst$$reg),
11440               as_Register($src1$$reg),
11441               as_Register($src2$$reg),
11442               Assembler::ROR,
11443               $src3$$constant & 0x1f);
11444   %}
11445 
11446   ins_pipe(ialu_reg_reg_shift);
11447 %}
11448 
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 // val | (-1 ^ (val ror shift)) ==> orn
11452 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11453                          iRegL src1, iRegL src2,
11454                          immI src3, immL_M1 src4) %{
11455   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11456   ins_cost(1.9 * INSN_COST);
11457   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11458 
11459   ins_encode %{
11460     __ orn(as_Register($dst$$reg),
11461               as_Register($src1$$reg),
11462               as_Register($src2$$reg),
11463               Assembler::ROR,
11464               $src3$$constant & 0x3f);
11465   %}
11466 
11467   ins_pipe(ialu_reg_reg_shift);
11468 %}
11469 
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 // val | (-1 ^ (val << shift)) ==> ornw
11473 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11474                          iRegIorL2I src1, iRegIorL2I src2,
11475                          immI src3, immI_M1 src4) %{
11476   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11477   ins_cost(1.9 * INSN_COST);
11478   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11479 
11480   ins_encode %{
11481     __ ornw(as_Register($dst$$reg),
11482               as_Register($src1$$reg),
11483               as_Register($src2$$reg),
11484               Assembler::LSL,
11485               $src3$$constant & 0x1f);
11486   %}
11487 
11488   ins_pipe(ialu_reg_reg_shift);
11489 %}
11490 
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 // val | (-1 ^ (val << shift)) ==> orn
11494 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11495                          iRegL src1, iRegL src2,
11496                          immI src3, immL_M1 src4) %{
11497   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11498   ins_cost(1.9 * INSN_COST);
11499   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11500 
11501   ins_encode %{
11502     __ orn(as_Register($dst$$reg),
11503               as_Register($src1$$reg),
11504               as_Register($src2$$reg),
11505               Assembler::LSL,
11506               $src3$$constant & 0x3f);
11507   %}
11508 
11509   ins_pipe(ialu_reg_reg_shift);
11510 %}
11511 
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11515                          iRegIorL2I src1, iRegIorL2I src2,
11516                          immI src3) %{
11517   match(Set dst (AndI src1 (URShiftI src2 src3)));
11518 
11519   ins_cost(1.9 * INSN_COST);
11520   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11521 
11522   ins_encode %{
11523     __ andw(as_Register($dst$$reg),
11524               as_Register($src1$$reg),
11525               as_Register($src2$$reg),
11526               Assembler::LSR,
11527               $src3$$constant & 0x1f);
11528   %}
11529 
11530   ins_pipe(ialu_reg_reg_shift);
11531 %}
11532 
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11536                          iRegL src1, iRegL src2,
11537                          immI src3) %{
11538   match(Set dst (AndL src1 (URShiftL src2 src3)));
11539 
11540   ins_cost(1.9 * INSN_COST);
11541   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11542 
11543   ins_encode %{
11544     __ andr(as_Register($dst$$reg),
11545               as_Register($src1$$reg),
11546               as_Register($src2$$reg),
11547               Assembler::LSR,
11548               $src3$$constant & 0x3f);
11549   %}
11550 
11551   ins_pipe(ialu_reg_reg_shift);
11552 %}
11553 
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11557                          iRegIorL2I src1, iRegIorL2I src2,
11558                          immI src3) %{
11559   match(Set dst (AndI src1 (RShiftI src2 src3)));
11560 
11561   ins_cost(1.9 * INSN_COST);
11562   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11563 
11564   ins_encode %{
11565     __ andw(as_Register($dst$$reg),
11566               as_Register($src1$$reg),
11567               as_Register($src2$$reg),
11568               Assembler::ASR,
11569               $src3$$constant & 0x1f);
11570   %}
11571 
11572   ins_pipe(ialu_reg_reg_shift);
11573 %}
11574 
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11578                          iRegL src1, iRegL src2,
11579                          immI src3) %{
11580   match(Set dst (AndL src1 (RShiftL src2 src3)));
11581 
11582   ins_cost(1.9 * INSN_COST);
11583   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11584 
11585   ins_encode %{
11586     __ andr(as_Register($dst$$reg),
11587               as_Register($src1$$reg),
11588               as_Register($src2$$reg),
11589               Assembler::ASR,
11590               $src3$$constant & 0x3f);
11591   %}
11592 
11593   ins_pipe(ialu_reg_reg_shift);
11594 %}
11595 
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11599                          iRegIorL2I src1, iRegIorL2I src2,
11600                          immI src3) %{
11601   match(Set dst (AndI src1 (LShiftI src2 src3)));
11602 
11603   ins_cost(1.9 * INSN_COST);
11604   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11605 
11606   ins_encode %{
11607     __ andw(as_Register($dst$$reg),
11608               as_Register($src1$$reg),
11609               as_Register($src2$$reg),
11610               Assembler::LSL,
11611               $src3$$constant & 0x1f);
11612   %}
11613 
11614   ins_pipe(ialu_reg_reg_shift);
11615 %}
11616 
11617 // This pattern is automatically generated from aarch64_ad.m4
11618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11619 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11620                          iRegL src1, iRegL src2,
11621                          immI src3) %{
11622   match(Set dst (AndL src1 (LShiftL src2 src3)));
11623 
11624   ins_cost(1.9 * INSN_COST);
11625   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11626 
11627   ins_encode %{
11628     __ andr(as_Register($dst$$reg),
11629               as_Register($src1$$reg),
11630               as_Register($src2$$reg),
11631               Assembler::LSL,
11632               $src3$$constant & 0x3f);
11633   %}
11634 
11635   ins_pipe(ialu_reg_reg_shift);
11636 %}
11637 
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11641                          iRegIorL2I src1, iRegIorL2I src2,
11642                          immI src3) %{
11643   match(Set dst (AndI src1 (RotateRight src2 src3)));
11644 
11645   ins_cost(1.9 * INSN_COST);
11646   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11647 
11648   ins_encode %{
11649     __ andw(as_Register($dst$$reg),
11650               as_Register($src1$$reg),
11651               as_Register($src2$$reg),
11652               Assembler::ROR,
11653               $src3$$constant & 0x1f);
11654   %}
11655 
11656   ins_pipe(ialu_reg_reg_shift);
11657 %}
11658 
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11662                          iRegL src1, iRegL src2,
11663                          immI src3) %{
11664   match(Set dst (AndL src1 (RotateRight src2 src3)));
11665 
11666   ins_cost(1.9 * INSN_COST);
11667   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11668 
11669   ins_encode %{
11670     __ andr(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::ROR,
11674               $src3$$constant & 0x3f);
11675   %}
11676 
11677   ins_pipe(ialu_reg_reg_shift);
11678 %}
11679 
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11683                          iRegIorL2I src1, iRegIorL2I src2,
11684                          immI src3) %{
11685   match(Set dst (XorI src1 (URShiftI src2 src3)));
11686 
11687   ins_cost(1.9 * INSN_COST);
11688   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11689 
11690   ins_encode %{
11691     __ eorw(as_Register($dst$$reg),
11692               as_Register($src1$$reg),
11693               as_Register($src2$$reg),
11694               Assembler::LSR,
11695               $src3$$constant & 0x1f);
11696   %}
11697 
11698   ins_pipe(ialu_reg_reg_shift);
11699 %}
11700 
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11704                          iRegL src1, iRegL src2,
11705                          immI src3) %{
11706   match(Set dst (XorL src1 (URShiftL src2 src3)));
11707 
11708   ins_cost(1.9 * INSN_COST);
11709   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11710 
11711   ins_encode %{
11712     __ eor(as_Register($dst$$reg),
11713               as_Register($src1$$reg),
11714               as_Register($src2$$reg),
11715               Assembler::LSR,
11716               $src3$$constant & 0x3f);
11717   %}
11718 
11719   ins_pipe(ialu_reg_reg_shift);
11720 %}
11721 
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11725                          iRegIorL2I src1, iRegIorL2I src2,
11726                          immI src3) %{
11727   match(Set dst (XorI src1 (RShiftI src2 src3)));
11728 
11729   ins_cost(1.9 * INSN_COST);
11730   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11731 
11732   ins_encode %{
11733     __ eorw(as_Register($dst$$reg),
11734               as_Register($src1$$reg),
11735               as_Register($src2$$reg),
11736               Assembler::ASR,
11737               $src3$$constant & 0x1f);
11738   %}
11739 
11740   ins_pipe(ialu_reg_reg_shift);
11741 %}
11742 
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11746                          iRegL src1, iRegL src2,
11747                          immI src3) %{
11748   match(Set dst (XorL src1 (RShiftL src2 src3)));
11749 
11750   ins_cost(1.9 * INSN_COST);
11751   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11752 
11753   ins_encode %{
11754     __ eor(as_Register($dst$$reg),
11755               as_Register($src1$$reg),
11756               as_Register($src2$$reg),
11757               Assembler::ASR,
11758               $src3$$constant & 0x3f);
11759   %}
11760 
11761   ins_pipe(ialu_reg_reg_shift);
11762 %}
11763 
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11767                          iRegIorL2I src1, iRegIorL2I src2,
11768                          immI src3) %{
11769   match(Set dst (XorI src1 (LShiftI src2 src3)));
11770 
11771   ins_cost(1.9 * INSN_COST);
11772   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11773 
11774   ins_encode %{
11775     __ eorw(as_Register($dst$$reg),
11776               as_Register($src1$$reg),
11777               as_Register($src2$$reg),
11778               Assembler::LSL,
11779               $src3$$constant & 0x1f);
11780   %}
11781 
11782   ins_pipe(ialu_reg_reg_shift);
11783 %}
11784 
11785 // This pattern is automatically generated from aarch64_ad.m4
11786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11787 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11788                          iRegL src1, iRegL src2,
11789                          immI src3) %{
11790   match(Set dst (XorL src1 (LShiftL src2 src3)));
11791 
11792   ins_cost(1.9 * INSN_COST);
11793   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11794 
11795   ins_encode %{
11796     __ eor(as_Register($dst$$reg),
11797               as_Register($src1$$reg),
11798               as_Register($src2$$reg),
11799               Assembler::LSL,
11800               $src3$$constant & 0x3f);
11801   %}
11802 
11803   ins_pipe(ialu_reg_reg_shift);
11804 %}
11805 
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11809                          iRegIorL2I src1, iRegIorL2I src2,
11810                          immI src3) %{
11811   match(Set dst (XorI src1 (RotateRight src2 src3)));
11812 
11813   ins_cost(1.9 * INSN_COST);
11814   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11815 
11816   ins_encode %{
11817     __ eorw(as_Register($dst$$reg),
11818               as_Register($src1$$reg),
11819               as_Register($src2$$reg),
11820               Assembler::ROR,
11821               $src3$$constant & 0x1f);
11822   %}
11823 
11824   ins_pipe(ialu_reg_reg_shift);
11825 %}
11826 
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11830                          iRegL src1, iRegL src2,
11831                          immI src3) %{
11832   match(Set dst (XorL src1 (RotateRight src2 src3)));
11833 
11834   ins_cost(1.9 * INSN_COST);
11835   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11836 
11837   ins_encode %{
11838     __ eor(as_Register($dst$$reg),
11839               as_Register($src1$$reg),
11840               as_Register($src2$$reg),
11841               Assembler::ROR,
11842               $src3$$constant & 0x3f);
11843   %}
11844 
11845   ins_pipe(ialu_reg_reg_shift);
11846 %}
11847 
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11851                          iRegIorL2I src1, iRegIorL2I src2,
11852                          immI src3) %{
11853   match(Set dst (OrI src1 (URShiftI src2 src3)));
11854 
11855   ins_cost(1.9 * INSN_COST);
11856   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11857 
11858   ins_encode %{
11859     __ orrw(as_Register($dst$$reg),
11860               as_Register($src1$$reg),
11861               as_Register($src2$$reg),
11862               Assembler::LSR,
11863               $src3$$constant & 0x1f);
11864   %}
11865 
11866   ins_pipe(ialu_reg_reg_shift);
11867 %}
11868 
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11872                          iRegL src1, iRegL src2,
11873                          immI src3) %{
11874   match(Set dst (OrL src1 (URShiftL src2 src3)));
11875 
11876   ins_cost(1.9 * INSN_COST);
11877   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11878 
11879   ins_encode %{
11880     __ orr(as_Register($dst$$reg),
11881               as_Register($src1$$reg),
11882               as_Register($src2$$reg),
11883               Assembler::LSR,
11884               $src3$$constant & 0x3f);
11885   %}
11886 
11887   ins_pipe(ialu_reg_reg_shift);
11888 %}
11889 
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11893                          iRegIorL2I src1, iRegIorL2I src2,
11894                          immI src3) %{
11895   match(Set dst (OrI src1 (RShiftI src2 src3)));
11896 
11897   ins_cost(1.9 * INSN_COST);
11898   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11899 
11900   ins_encode %{
11901     __ orrw(as_Register($dst$$reg),
11902               as_Register($src1$$reg),
11903               as_Register($src2$$reg),
11904               Assembler::ASR,
11905               $src3$$constant & 0x1f);
11906   %}
11907 
11908   ins_pipe(ialu_reg_reg_shift);
11909 %}
11910 
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11914                          iRegL src1, iRegL src2,
11915                          immI src3) %{
11916   match(Set dst (OrL src1 (RShiftL src2 src3)));
11917 
11918   ins_cost(1.9 * INSN_COST);
11919   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11920 
11921   ins_encode %{
11922     __ orr(as_Register($dst$$reg),
11923               as_Register($src1$$reg),
11924               as_Register($src2$$reg),
11925               Assembler::ASR,
11926               $src3$$constant & 0x3f);
11927   %}
11928 
11929   ins_pipe(ialu_reg_reg_shift);
11930 %}
11931 
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11935                          iRegIorL2I src1, iRegIorL2I src2,
11936                          immI src3) %{
11937   match(Set dst (OrI src1 (LShiftI src2 src3)));
11938 
11939   ins_cost(1.9 * INSN_COST);
11940   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11941 
11942   ins_encode %{
11943     __ orrw(as_Register($dst$$reg),
11944               as_Register($src1$$reg),
11945               as_Register($src2$$reg),
11946               Assembler::LSL,
11947               $src3$$constant & 0x1f);
11948   %}
11949 
11950   ins_pipe(ialu_reg_reg_shift);
11951 %}
11952 
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11956                          iRegL src1, iRegL src2,
11957                          immI src3) %{
11958   match(Set dst (OrL src1 (LShiftL src2 src3)));
11959 
11960   ins_cost(1.9 * INSN_COST);
11961   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11962 
11963   ins_encode %{
11964     __ orr(as_Register($dst$$reg),
11965               as_Register($src1$$reg),
11966               as_Register($src2$$reg),
11967               Assembler::LSL,
11968               $src3$$constant & 0x3f);
11969   %}
11970 
11971   ins_pipe(ialu_reg_reg_shift);
11972 %}
11973 
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11977                          iRegIorL2I src1, iRegIorL2I src2,
11978                          immI src3) %{
11979   match(Set dst (OrI src1 (RotateRight src2 src3)));
11980 
11981   ins_cost(1.9 * INSN_COST);
11982   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
11983 
11984   ins_encode %{
11985     __ orrw(as_Register($dst$$reg),
11986               as_Register($src1$$reg),
11987               as_Register($src2$$reg),
11988               Assembler::ROR,
11989               $src3$$constant & 0x1f);
11990   %}
11991 
11992   ins_pipe(ialu_reg_reg_shift);
11993 %}
11994 
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11998                          iRegL src1, iRegL src2,
11999                          immI src3) %{
12000   match(Set dst (OrL src1 (RotateRight src2 src3)));
12001 
12002   ins_cost(1.9 * INSN_COST);
12003   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12004 
12005   ins_encode %{
12006     __ orr(as_Register($dst$$reg),
12007               as_Register($src1$$reg),
12008               as_Register($src2$$reg),
12009               Assembler::ROR,
12010               $src3$$constant & 0x3f);
12011   %}
12012 
12013   ins_pipe(ialu_reg_reg_shift);
12014 %}
12015 
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12019                          iRegIorL2I src1, iRegIorL2I src2,
12020                          immI src3) %{
12021   match(Set dst (AddI src1 (URShiftI src2 src3)));
12022 
12023   ins_cost(1.9 * INSN_COST);
12024   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12025 
12026   ins_encode %{
12027     __ addw(as_Register($dst$$reg),
12028               as_Register($src1$$reg),
12029               as_Register($src2$$reg),
12030               Assembler::LSR,
12031               $src3$$constant & 0x1f);
12032   %}
12033 
12034   ins_pipe(ialu_reg_reg_shift);
12035 %}
12036 
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12040                          iRegL src1, iRegL src2,
12041                          immI src3) %{
12042   match(Set dst (AddL src1 (URShiftL src2 src3)));
12043 
12044   ins_cost(1.9 * INSN_COST);
12045   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12046 
12047   ins_encode %{
12048     __ add(as_Register($dst$$reg),
12049               as_Register($src1$$reg),
12050               as_Register($src2$$reg),
12051               Assembler::LSR,
12052               $src3$$constant & 0x3f);
12053   %}
12054 
12055   ins_pipe(ialu_reg_reg_shift);
12056 %}
12057 
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12061                          iRegIorL2I src1, iRegIorL2I src2,
12062                          immI src3) %{
12063   match(Set dst (AddI src1 (RShiftI src2 src3)));
12064 
12065   ins_cost(1.9 * INSN_COST);
12066   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12067 
12068   ins_encode %{
12069     __ addw(as_Register($dst$$reg),
12070               as_Register($src1$$reg),
12071               as_Register($src2$$reg),
12072               Assembler::ASR,
12073               $src3$$constant & 0x1f);
12074   %}
12075 
12076   ins_pipe(ialu_reg_reg_shift);
12077 %}
12078 
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12082                          iRegL src1, iRegL src2,
12083                          immI src3) %{
12084   match(Set dst (AddL src1 (RShiftL src2 src3)));
12085 
12086   ins_cost(1.9 * INSN_COST);
12087   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12088 
12089   ins_encode %{
12090     __ add(as_Register($dst$$reg),
12091               as_Register($src1$$reg),
12092               as_Register($src2$$reg),
12093               Assembler::ASR,
12094               $src3$$constant & 0x3f);
12095   %}
12096 
12097   ins_pipe(ialu_reg_reg_shift);
12098 %}
12099 
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12103                          iRegIorL2I src1, iRegIorL2I src2,
12104                          immI src3) %{
12105   match(Set dst (AddI src1 (LShiftI src2 src3)));
12106 
12107   ins_cost(1.9 * INSN_COST);
12108   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12109 
12110   ins_encode %{
12111     __ addw(as_Register($dst$$reg),
12112               as_Register($src1$$reg),
12113               as_Register($src2$$reg),
12114               Assembler::LSL,
12115               $src3$$constant & 0x1f);
12116   %}
12117 
12118   ins_pipe(ialu_reg_reg_shift);
12119 %}
12120 
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12124                          iRegL src1, iRegL src2,
12125                          immI src3) %{
12126   match(Set dst (AddL src1 (LShiftL src2 src3)));
12127 
12128   ins_cost(1.9 * INSN_COST);
12129   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12130 
12131   ins_encode %{
12132     __ add(as_Register($dst$$reg),
12133               as_Register($src1$$reg),
12134               as_Register($src2$$reg),
12135               Assembler::LSL,
12136               $src3$$constant & 0x3f);
12137   %}
12138 
12139   ins_pipe(ialu_reg_reg_shift);
12140 %}
12141 
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12145                          iRegIorL2I src1, iRegIorL2I src2,
12146                          immI src3) %{
12147   match(Set dst (SubI src1 (URShiftI src2 src3)));
12148 
12149   ins_cost(1.9 * INSN_COST);
12150   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12151 
12152   ins_encode %{
12153     __ subw(as_Register($dst$$reg),
12154               as_Register($src1$$reg),
12155               as_Register($src2$$reg),
12156               Assembler::LSR,
12157               $src3$$constant & 0x1f);
12158   %}
12159 
12160   ins_pipe(ialu_reg_reg_shift);
12161 %}
12162 
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12166                          iRegL src1, iRegL src2,
12167                          immI src3) %{
12168   match(Set dst (SubL src1 (URShiftL src2 src3)));
12169 
12170   ins_cost(1.9 * INSN_COST);
12171   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12172 
12173   ins_encode %{
12174     __ sub(as_Register($dst$$reg),
12175               as_Register($src1$$reg),
12176               as_Register($src2$$reg),
12177               Assembler::LSR,
12178               $src3$$constant & 0x3f);
12179   %}
12180 
12181   ins_pipe(ialu_reg_reg_shift);
12182 %}
12183 
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12187                          iRegIorL2I src1, iRegIorL2I src2,
12188                          immI src3) %{
12189   match(Set dst (SubI src1 (RShiftI src2 src3)));
12190 
12191   ins_cost(1.9 * INSN_COST);
12192   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12193 
12194   ins_encode %{
12195     __ subw(as_Register($dst$$reg),
12196               as_Register($src1$$reg),
12197               as_Register($src2$$reg),
12198               Assembler::ASR,
12199               $src3$$constant & 0x1f);
12200   %}
12201 
12202   ins_pipe(ialu_reg_reg_shift);
12203 %}
12204 
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12208                          iRegL src1, iRegL src2,
12209                          immI src3) %{
12210   match(Set dst (SubL src1 (RShiftL src2 src3)));
12211 
12212   ins_cost(1.9 * INSN_COST);
12213   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12214 
12215   ins_encode %{
12216     __ sub(as_Register($dst$$reg),
12217               as_Register($src1$$reg),
12218               as_Register($src2$$reg),
12219               Assembler::ASR,
12220               $src3$$constant & 0x3f);
12221   %}
12222 
12223   ins_pipe(ialu_reg_reg_shift);
12224 %}
12225 
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12229                          iRegIorL2I src1, iRegIorL2I src2,
12230                          immI src3) %{
12231   match(Set dst (SubI src1 (LShiftI src2 src3)));
12232 
12233   ins_cost(1.9 * INSN_COST);
12234   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12235 
12236   ins_encode %{
12237     __ subw(as_Register($dst$$reg),
12238               as_Register($src1$$reg),
12239               as_Register($src2$$reg),
12240               Assembler::LSL,
12241               $src3$$constant & 0x1f);
12242   %}
12243 
12244   ins_pipe(ialu_reg_reg_shift);
12245 %}
12246 
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12250                          iRegL src1, iRegL src2,
12251                          immI src3) %{
12252   match(Set dst (SubL src1 (LShiftL src2 src3)));
12253 
12254   ins_cost(1.9 * INSN_COST);
12255   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12256 
12257   ins_encode %{
12258     __ sub(as_Register($dst$$reg),
12259               as_Register($src1$$reg),
12260               as_Register($src2$$reg),
12261               Assembler::LSL,
12262               $src3$$constant & 0x3f);
12263   %}
12264 
12265   ins_pipe(ialu_reg_reg_shift);
12266 %}
12267 
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 
12271 // Shift Left followed by Shift Right.
12272 // This idiom is used by the compiler for the i2b bytecode etc.
12273 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12274 %{
12275   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12276   ins_cost(INSN_COST * 2);
12277   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12278   ins_encode %{
12279     int lshift = $lshift_count$$constant & 63;
12280     int rshift = $rshift_count$$constant & 63;
12281     int s = 63 - lshift;
12282     int r = (rshift - lshift) & 63;
12283     __ sbfm(as_Register($dst$$reg),
12284             as_Register($src$$reg),
12285             r, s);
12286   %}
12287 
12288   ins_pipe(ialu_reg_shift);
12289 %}
12290 
12291 // This pattern is automatically generated from aarch64_ad.m4
12292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12293 
12294 // Shift Left followed by Shift Right.
12295 // This idiom is used by the compiler for the i2b bytecode etc.
12296 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12297 %{
12298   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12299   ins_cost(INSN_COST * 2);
12300   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12301   ins_encode %{
12302     int lshift = $lshift_count$$constant & 31;
12303     int rshift = $rshift_count$$constant & 31;
12304     int s = 31 - lshift;
12305     int r = (rshift - lshift) & 31;
12306     __ sbfmw(as_Register($dst$$reg),
12307             as_Register($src$$reg),
12308             r, s);
12309   %}
12310 
12311   ins_pipe(ialu_reg_shift);
12312 %}
12313 
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 
12317 // Shift Left followed by Shift Right.
12318 // This idiom is used by the compiler for the i2b bytecode etc.
12319 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12320 %{
12321   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12322   ins_cost(INSN_COST * 2);
12323   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12324   ins_encode %{
12325     int lshift = $lshift_count$$constant & 63;
12326     int rshift = $rshift_count$$constant & 63;
12327     int s = 63 - lshift;
12328     int r = (rshift - lshift) & 63;
12329     __ ubfm(as_Register($dst$$reg),
12330             as_Register($src$$reg),
12331             r, s);
12332   %}
12333 
12334   ins_pipe(ialu_reg_shift);
12335 %}
12336 
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 
12340 // Shift Left followed by Shift Right.
12341 // This idiom is used by the compiler for the i2b bytecode etc.
12342 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12343 %{
12344   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12345   ins_cost(INSN_COST * 2);
12346   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12347   ins_encode %{
12348     int lshift = $lshift_count$$constant & 31;
12349     int rshift = $rshift_count$$constant & 31;
12350     int s = 31 - lshift;
12351     int r = (rshift - lshift) & 31;
12352     __ ubfmw(as_Register($dst$$reg),
12353             as_Register($src$$reg),
12354             r, s);
12355   %}
12356 
12357   ins_pipe(ialu_reg_shift);
12358 %}
12359 
12360 // Bitfield extract with shift & mask
12361 
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12365 %{
12366   match(Set dst (AndI (URShiftI src rshift) mask));
12367   // Make sure we are not going to exceed what ubfxw can do.
12368   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12369 
12370   ins_cost(INSN_COST);
12371   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12372   ins_encode %{
12373     int rshift = $rshift$$constant & 31;
12374     intptr_t mask = $mask$$constant;
12375     int width = exact_log2(mask+1);
12376     __ ubfxw(as_Register($dst$$reg),
12377             as_Register($src$$reg), rshift, width);
12378   %}
12379   ins_pipe(ialu_reg_shift);
12380 %}
12381 
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12385 %{
12386   match(Set dst (AndL (URShiftL src rshift) mask));
12387   // Make sure we are not going to exceed what ubfx can do.
12388   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12389 
12390   ins_cost(INSN_COST);
12391   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12392   ins_encode %{
12393     int rshift = $rshift$$constant & 63;
12394     intptr_t mask = $mask$$constant;
12395     int width = exact_log2_long(mask+1);
12396     __ ubfx(as_Register($dst$$reg),
12397             as_Register($src$$reg), rshift, width);
12398   %}
12399   ins_pipe(ialu_reg_shift);
12400 %}
12401 
12402 
12403 // This pattern is automatically generated from aarch64_ad.m4
12404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12405 
12406 // We can use ubfx when extending an And with a mask when we know mask
12407 // is positive.  We know that because immI_bitmask guarantees it.
12408 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12409 %{
12410   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12411   // Make sure we are not going to exceed what ubfxw can do.
12412   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12413 
12414   ins_cost(INSN_COST * 2);
12415   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12416   ins_encode %{
12417     int rshift = $rshift$$constant & 31;
12418     intptr_t mask = $mask$$constant;
12419     int width = exact_log2(mask+1);
12420     __ ubfx(as_Register($dst$$reg),
12421             as_Register($src$$reg), rshift, width);
12422   %}
12423   ins_pipe(ialu_reg_shift);
12424 %}
12425 
12426 
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 
12430 // We can use ubfiz when masking by a positive number and then left shifting the result.
12431 // We know that the mask is positive because immI_bitmask guarantees it.
12432 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12433 %{
12434   match(Set dst (LShiftI (AndI src mask) lshift));
12435   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12436 
12437   ins_cost(INSN_COST);
12438   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12439   ins_encode %{
12440     int lshift = $lshift$$constant & 31;
12441     intptr_t mask = $mask$$constant;
12442     int width = exact_log2(mask+1);
12443     __ ubfizw(as_Register($dst$$reg),
12444           as_Register($src$$reg), lshift, width);
12445   %}
12446   ins_pipe(ialu_reg_shift);
12447 %}
12448 
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 
12452 // We can use ubfiz when masking by a positive number and then left shifting the result.
12453 // We know that the mask is positive because immL_bitmask guarantees it.
12454 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12455 %{
12456   match(Set dst (LShiftL (AndL src mask) lshift));
12457   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12458 
12459   ins_cost(INSN_COST);
12460   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12461   ins_encode %{
12462     int lshift = $lshift$$constant & 63;
12463     intptr_t mask = $mask$$constant;
12464     int width = exact_log2_long(mask+1);
12465     __ ubfiz(as_Register($dst$$reg),
12466           as_Register($src$$reg), lshift, width);
12467   %}
12468   ins_pipe(ialu_reg_shift);
12469 %}
12470 
12471 // This pattern is automatically generated from aarch64_ad.m4
12472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12473 
12474 // We can use ubfiz when masking by a positive number and then left shifting the result.
12475 // We know that the mask is positive because immI_bitmask guarantees it.
12476 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12477 %{
12478   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12479   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12480 
12481   ins_cost(INSN_COST);
12482   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12483   ins_encode %{
12484     int lshift = $lshift$$constant & 31;
12485     intptr_t mask = $mask$$constant;
12486     int width = exact_log2(mask+1);
12487     __ ubfizw(as_Register($dst$$reg),
12488           as_Register($src$$reg), lshift, width);
12489   %}
12490   ins_pipe(ialu_reg_shift);
12491 %}
12492 
12493 // This pattern is automatically generated from aarch64_ad.m4
12494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12495 
12496 // We can use ubfiz when masking by a positive number and then left shifting the result.
12497 // We know that the mask is positive because immL_bitmask guarantees it.
12498 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12499 %{
12500   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12501   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12502 
12503   ins_cost(INSN_COST);
12504   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12505   ins_encode %{
12506     int lshift = $lshift$$constant & 63;
12507     intptr_t mask = $mask$$constant;
12508     int width = exact_log2_long(mask+1);
12509     __ ubfiz(as_Register($dst$$reg),
12510           as_Register($src$$reg), lshift, width);
12511   %}
12512   ins_pipe(ialu_reg_shift);
12513 %}
12514 
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 
12519 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12520 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12521 %{
12522   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12523   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12524 
12525   ins_cost(INSN_COST);
12526   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12527   ins_encode %{
12528     int lshift = $lshift$$constant & 63;
12529     intptr_t mask = $mask$$constant;
12530     int width = exact_log2(mask+1);
12531     __ ubfiz(as_Register($dst$$reg),
12532              as_Register($src$$reg), lshift, width);
12533   %}
12534   ins_pipe(ialu_reg_shift);
12535 %}
12536 
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539 
12540 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12541 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12542 %{
12543   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12544   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12545 
12546   ins_cost(INSN_COST);
12547   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12548   ins_encode %{
12549     int lshift = $lshift$$constant & 31;
12550     intptr_t mask = $mask$$constant;
12551     int width = exact_log2(mask+1);
12552     __ ubfiz(as_Register($dst$$reg),
12553              as_Register($src$$reg), lshift, width);
12554   %}
12555   ins_pipe(ialu_reg_shift);
12556 %}
12557 
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560 
12561 // Can skip int2long conversions after AND with small bitmask
12562 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12563 %{
12564   match(Set dst (ConvI2L (AndI src msk)));
12565   ins_cost(INSN_COST);
12566   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12567   ins_encode %{
12568     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12569   %}
12570   ins_pipe(ialu_reg_shift);
12571 %}
12572 
12573 
12574 // Rotations
12575 
12576 // This pattern is automatically generated from aarch64_ad.m4
12577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12578 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12579 %{
12580   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12581   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12582 
12583   ins_cost(INSN_COST);
12584   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12585 
12586   ins_encode %{
12587     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12588             $rshift$$constant & 63);
12589   %}
12590   ins_pipe(ialu_reg_reg_extr);
12591 %}
12592 
12593 
12594 // This pattern is automatically generated from aarch64_ad.m4
12595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12596 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12597 %{
12598   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12599   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12600 
12601   ins_cost(INSN_COST);
12602   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12603 
12604   ins_encode %{
12605     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12606             $rshift$$constant & 31);
12607   %}
12608   ins_pipe(ialu_reg_reg_extr);
12609 %}
12610 
12611 
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12615 %{
12616   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12617   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12618 
12619   ins_cost(INSN_COST);
12620   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12621 
12622   ins_encode %{
12623     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12624             $rshift$$constant & 63);
12625   %}
12626   ins_pipe(ialu_reg_reg_extr);
12627 %}
12628 
12629 
12630 // This pattern is automatically generated from aarch64_ad.m4
12631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12632 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12633 %{
12634   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12635   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12636 
12637   ins_cost(INSN_COST);
12638   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12639 
12640   ins_encode %{
12641     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12642             $rshift$$constant & 31);
12643   %}
12644   ins_pipe(ialu_reg_reg_extr);
12645 %}
12646 
12647 // This pattern is automatically generated from aarch64_ad.m4
12648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12649 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12650 %{
12651   match(Set dst (RotateRight src shift));
12652 
12653   ins_cost(INSN_COST);
12654   format %{ "ror    $dst, $src, $shift" %}
12655 
12656   ins_encode %{
12657      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12658                $shift$$constant & 0x1f);
12659   %}
12660   ins_pipe(ialu_reg_reg_vshift);
12661 %}
12662 
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12666 %{
12667   match(Set dst (RotateRight src shift));
12668 
12669   ins_cost(INSN_COST);
12670   format %{ "ror    $dst, $src, $shift" %}
12671 
12672   ins_encode %{
12673      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12674                $shift$$constant & 0x3f);
12675   %}
12676   ins_pipe(ialu_reg_reg_vshift);
12677 %}
12678 
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12682 %{
12683   match(Set dst (RotateRight src shift));
12684 
12685   ins_cost(INSN_COST);
12686   format %{ "ror    $dst, $src, $shift" %}
12687 
12688   ins_encode %{
12689      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12690   %}
12691   ins_pipe(ialu_reg_reg_vshift);
12692 %}
12693 
12694 // This pattern is automatically generated from aarch64_ad.m4
12695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12696 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12697 %{
12698   match(Set dst (RotateRight src shift));
12699 
12700   ins_cost(INSN_COST);
12701   format %{ "ror    $dst, $src, $shift" %}
12702 
12703   ins_encode %{
12704      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12705   %}
12706   ins_pipe(ialu_reg_reg_vshift);
12707 %}
12708 
12709 // This pattern is automatically generated from aarch64_ad.m4
12710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12711 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12712 %{
12713   match(Set dst (RotateLeft src shift));
12714 
12715   ins_cost(INSN_COST);
12716   format %{ "rol    $dst, $src, $shift" %}
12717 
12718   ins_encode %{
12719      __ subw(rscratch1, zr, as_Register($shift$$reg));
12720      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12721   %}
12722   ins_pipe(ialu_reg_reg_vshift);
12723 %}
12724 
12725 // This pattern is automatically generated from aarch64_ad.m4
12726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12727 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12728 %{
12729   match(Set dst (RotateLeft src shift));
12730 
12731   ins_cost(INSN_COST);
12732   format %{ "rol    $dst, $src, $shift" %}
12733 
12734   ins_encode %{
12735      __ subw(rscratch1, zr, as_Register($shift$$reg));
12736      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12737   %}
12738   ins_pipe(ialu_reg_reg_vshift);
12739 %}
12740 
12741 
12742 // Add/subtract (extended)
12743 
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12747 %{
12748   match(Set dst (AddL src1 (ConvI2L src2)));
12749   ins_cost(INSN_COST);
12750   format %{ "add  $dst, $src1, $src2, sxtw" %}
12751 
12752    ins_encode %{
12753      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12754             as_Register($src2$$reg), ext::sxtw);
12755    %}
12756   ins_pipe(ialu_reg_reg);
12757 %}
12758 
12759 // This pattern is automatically generated from aarch64_ad.m4
12760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12761 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12762 %{
12763   match(Set dst (SubL src1 (ConvI2L src2)));
12764   ins_cost(INSN_COST);
12765   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12766 
12767    ins_encode %{
12768      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12769             as_Register($src2$$reg), ext::sxtw);
12770    %}
12771   ins_pipe(ialu_reg_reg);
12772 %}
12773 
12774 // This pattern is automatically generated from aarch64_ad.m4
12775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12776 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12777 %{
12778   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12779   ins_cost(INSN_COST);
12780   format %{ "add  $dst, $src1, $src2, sxth" %}
12781 
12782    ins_encode %{
12783      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12784             as_Register($src2$$reg), ext::sxth);
12785    %}
12786   ins_pipe(ialu_reg_reg);
12787 %}
12788 
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12792 %{
12793   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12794   ins_cost(INSN_COST);
12795   format %{ "add  $dst, $src1, $src2, sxtb" %}
12796 
12797    ins_encode %{
12798      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12799             as_Register($src2$$reg), ext::sxtb);
12800    %}
12801   ins_pipe(ialu_reg_reg);
12802 %}
12803 
12804 // This pattern is automatically generated from aarch64_ad.m4
12805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12806 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12807 %{
12808   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12809   ins_cost(INSN_COST);
12810   format %{ "add  $dst, $src1, $src2, uxtb" %}
12811 
12812    ins_encode %{
12813      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12814             as_Register($src2$$reg), ext::uxtb);
12815    %}
12816   ins_pipe(ialu_reg_reg);
12817 %}
12818 
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12822 %{
12823   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12824   ins_cost(INSN_COST);
12825   format %{ "add  $dst, $src1, $src2, sxth" %}
12826 
12827    ins_encode %{
12828      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12829             as_Register($src2$$reg), ext::sxth);
12830    %}
12831   ins_pipe(ialu_reg_reg);
12832 %}
12833 
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12837 %{
12838   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12839   ins_cost(INSN_COST);
12840   format %{ "add  $dst, $src1, $src2, sxtw" %}
12841 
12842    ins_encode %{
12843      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12844             as_Register($src2$$reg), ext::sxtw);
12845    %}
12846   ins_pipe(ialu_reg_reg);
12847 %}
12848 
12849 // This pattern is automatically generated from aarch64_ad.m4
12850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12851 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12852 %{
12853   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12854   ins_cost(INSN_COST);
12855   format %{ "add  $dst, $src1, $src2, sxtb" %}
12856 
12857    ins_encode %{
12858      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12859             as_Register($src2$$reg), ext::sxtb);
12860    %}
12861   ins_pipe(ialu_reg_reg);
12862 %}
12863 
12864 // This pattern is automatically generated from aarch64_ad.m4
12865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12866 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12867 %{
12868   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12869   ins_cost(INSN_COST);
12870   format %{ "add  $dst, $src1, $src2, uxtb" %}
12871 
12872    ins_encode %{
12873      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12874             as_Register($src2$$reg), ext::uxtb);
12875    %}
12876   ins_pipe(ialu_reg_reg);
12877 %}
12878 
12879 // This pattern is automatically generated from aarch64_ad.m4
12880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12881 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12882 %{
12883   match(Set dst (AddI src1 (AndI src2 mask)));
12884   ins_cost(INSN_COST);
12885   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12886 
12887    ins_encode %{
12888      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12889             as_Register($src2$$reg), ext::uxtb);
12890    %}
12891   ins_pipe(ialu_reg_reg);
12892 %}
12893 
12894 // This pattern is automatically generated from aarch64_ad.m4
12895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12897 %{
12898   match(Set dst (AddI src1 (AndI src2 mask)));
12899   ins_cost(INSN_COST);
12900   format %{ "addw  $dst, $src1, $src2, uxth" %}
12901 
12902    ins_encode %{
12903      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12904             as_Register($src2$$reg), ext::uxth);
12905    %}
12906   ins_pipe(ialu_reg_reg);
12907 %}
12908 
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12912 %{
12913   match(Set dst (AddL src1 (AndL src2 mask)));
12914   ins_cost(INSN_COST);
12915   format %{ "add  $dst, $src1, $src2, uxtb" %}
12916 
12917    ins_encode %{
12918      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12919             as_Register($src2$$reg), ext::uxtb);
12920    %}
12921   ins_pipe(ialu_reg_reg);
12922 %}
12923 
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12927 %{
12928   match(Set dst (AddL src1 (AndL src2 mask)));
12929   ins_cost(INSN_COST);
12930   format %{ "add  $dst, $src1, $src2, uxth" %}
12931 
12932    ins_encode %{
12933      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12934             as_Register($src2$$reg), ext::uxth);
12935    %}
12936   ins_pipe(ialu_reg_reg);
12937 %}
12938 
12939 // This pattern is automatically generated from aarch64_ad.m4
12940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12941 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12942 %{
12943   match(Set dst (AddL src1 (AndL src2 mask)));
12944   ins_cost(INSN_COST);
12945   format %{ "add  $dst, $src1, $src2, uxtw" %}
12946 
12947    ins_encode %{
12948      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12949             as_Register($src2$$reg), ext::uxtw);
12950    %}
12951   ins_pipe(ialu_reg_reg);
12952 %}
12953 
12954 // This pattern is automatically generated from aarch64_ad.m4
12955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12956 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12957 %{
12958   match(Set dst (SubI src1 (AndI src2 mask)));
12959   ins_cost(INSN_COST);
12960   format %{ "subw  $dst, $src1, $src2, uxtb" %}
12961 
12962    ins_encode %{
12963      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12964             as_Register($src2$$reg), ext::uxtb);
12965    %}
12966   ins_pipe(ialu_reg_reg);
12967 %}
12968 
12969 // This pattern is automatically generated from aarch64_ad.m4
12970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12971 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12972 %{
12973   match(Set dst (SubI src1 (AndI src2 mask)));
12974   ins_cost(INSN_COST);
12975   format %{ "subw  $dst, $src1, $src2, uxth" %}
12976 
12977    ins_encode %{
12978      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12979             as_Register($src2$$reg), ext::uxth);
12980    %}
12981   ins_pipe(ialu_reg_reg);
12982 %}
12983 
12984 // This pattern is automatically generated from aarch64_ad.m4
12985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12986 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12987 %{
12988   match(Set dst (SubL src1 (AndL src2 mask)));
12989   ins_cost(INSN_COST);
12990   format %{ "sub  $dst, $src1, $src2, uxtb" %}
12991 
12992    ins_encode %{
12993      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12994             as_Register($src2$$reg), ext::uxtb);
12995    %}
12996   ins_pipe(ialu_reg_reg);
12997 %}
12998 
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13002 %{
13003   match(Set dst (SubL src1 (AndL src2 mask)));
13004   ins_cost(INSN_COST);
13005   format %{ "sub  $dst, $src1, $src2, uxth" %}
13006 
13007    ins_encode %{
13008      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13009             as_Register($src2$$reg), ext::uxth);
13010    %}
13011   ins_pipe(ialu_reg_reg);
13012 %}
13013 
13014 // This pattern is automatically generated from aarch64_ad.m4
13015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13016 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13017 %{
13018   match(Set dst (SubL src1 (AndL src2 mask)));
13019   ins_cost(INSN_COST);
13020   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13021 
13022    ins_encode %{
13023      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13024             as_Register($src2$$reg), ext::uxtw);
13025    %}
13026   ins_pipe(ialu_reg_reg);
13027 %}
13028 
13029 
13030 // This pattern is automatically generated from aarch64_ad.m4
13031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13032 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13033 %{
13034   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13035   ins_cost(1.9 * INSN_COST);
13036   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13037 
13038    ins_encode %{
13039      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13040             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13041    %}
13042   ins_pipe(ialu_reg_reg_shift);
13043 %}
13044 
13045 // This pattern is automatically generated from aarch64_ad.m4
13046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13047 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13048 %{
13049   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13050   ins_cost(1.9 * INSN_COST);
13051   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13052 
13053    ins_encode %{
13054      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13055             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13056    %}
13057   ins_pipe(ialu_reg_reg_shift);
13058 %}
13059 
13060 // This pattern is automatically generated from aarch64_ad.m4
13061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13062 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13063 %{
13064   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13065   ins_cost(1.9 * INSN_COST);
13066   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13067 
13068    ins_encode %{
13069      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13070             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13071    %}
13072   ins_pipe(ialu_reg_reg_shift);
13073 %}
13074 
13075 // This pattern is automatically generated from aarch64_ad.m4
13076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13077 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13078 %{
13079   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13080   ins_cost(1.9 * INSN_COST);
13081   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13082 
13083    ins_encode %{
13084      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13085             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13086    %}
13087   ins_pipe(ialu_reg_reg_shift);
13088 %}
13089 
13090 // This pattern is automatically generated from aarch64_ad.m4
13091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13092 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13093 %{
13094   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13095   ins_cost(1.9 * INSN_COST);
13096   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13097 
13098    ins_encode %{
13099      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13100             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13101    %}
13102   ins_pipe(ialu_reg_reg_shift);
13103 %}
13104 
13105 // This pattern is automatically generated from aarch64_ad.m4
13106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13107 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13108 %{
13109   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13110   ins_cost(1.9 * INSN_COST);
13111   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13112 
13113    ins_encode %{
13114      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13115             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13116    %}
13117   ins_pipe(ialu_reg_reg_shift);
13118 %}
13119 
13120 // This pattern is automatically generated from aarch64_ad.m4
13121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13122 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13123 %{
13124   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13125   ins_cost(1.9 * INSN_COST);
13126   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13127 
13128    ins_encode %{
13129      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13130             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13131    %}
13132   ins_pipe(ialu_reg_reg_shift);
13133 %}
13134 
13135 // This pattern is automatically generated from aarch64_ad.m4
13136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13137 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13138 %{
13139   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13140   ins_cost(1.9 * INSN_COST);
13141   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13142 
13143    ins_encode %{
13144      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13145             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13146    %}
13147   ins_pipe(ialu_reg_reg_shift);
13148 %}
13149 
13150 // This pattern is automatically generated from aarch64_ad.m4
13151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13152 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13153 %{
13154   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13155   ins_cost(1.9 * INSN_COST);
13156   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13157 
13158    ins_encode %{
13159      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13160             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13161    %}
13162   ins_pipe(ialu_reg_reg_shift);
13163 %}
13164 
13165 // This pattern is automatically generated from aarch64_ad.m4
13166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13167 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13168 %{
13169   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13170   ins_cost(1.9 * INSN_COST);
13171   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13172 
13173    ins_encode %{
13174      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13175             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13176    %}
13177   ins_pipe(ialu_reg_reg_shift);
13178 %}
13179 
13180 // This pattern is automatically generated from aarch64_ad.m4
13181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13182 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13183 %{
13184   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13185   ins_cost(1.9 * INSN_COST);
13186   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13187 
13188    ins_encode %{
13189      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13190             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13191    %}
13192   ins_pipe(ialu_reg_reg_shift);
13193 %}
13194 
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13198 %{
13199   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13200   ins_cost(1.9 * INSN_COST);
13201   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13202 
13203    ins_encode %{
13204      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13205             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13206    %}
13207   ins_pipe(ialu_reg_reg_shift);
13208 %}
13209 
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13213 %{
13214   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13215   ins_cost(1.9 * INSN_COST);
13216   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13217 
13218    ins_encode %{
13219      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13220             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13221    %}
13222   ins_pipe(ialu_reg_reg_shift);
13223 %}
13224 
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13228 %{
13229   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13230   ins_cost(1.9 * INSN_COST);
13231   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13232 
13233    ins_encode %{
13234      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13235             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13236    %}
13237   ins_pipe(ialu_reg_reg_shift);
13238 %}
13239 
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13243 %{
13244   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13245   ins_cost(1.9 * INSN_COST);
13246   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13247 
13248    ins_encode %{
13249      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13250             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13251    %}
13252   ins_pipe(ialu_reg_reg_shift);
13253 %}
13254 
13255 // This pattern is automatically generated from aarch64_ad.m4
13256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13257 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13258 %{
13259   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13260   ins_cost(1.9 * INSN_COST);
13261   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13262 
13263    ins_encode %{
13264      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13265             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13266    %}
13267   ins_pipe(ialu_reg_reg_shift);
13268 %}
13269 
13270 // This pattern is automatically generated from aarch64_ad.m4
13271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13272 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13273 %{
13274   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13275   ins_cost(1.9 * INSN_COST);
13276   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13277 
13278    ins_encode %{
13279      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13280             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13281    %}
13282   ins_pipe(ialu_reg_reg_shift);
13283 %}
13284 
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13288 %{
13289   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13290   ins_cost(1.9 * INSN_COST);
13291   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13292 
13293    ins_encode %{
13294      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13295             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13296    %}
13297   ins_pipe(ialu_reg_reg_shift);
13298 %}
13299 
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13303 %{
13304   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13305   ins_cost(1.9 * INSN_COST);
13306   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13307 
13308    ins_encode %{
13309      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13310             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13311    %}
13312   ins_pipe(ialu_reg_reg_shift);
13313 %}
13314 
13315 // This pattern is automatically generated from aarch64_ad.m4
13316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13317 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13318 %{
13319   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13320   ins_cost(1.9 * INSN_COST);
13321   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13322 
13323    ins_encode %{
13324      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13325             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13326    %}
13327   ins_pipe(ialu_reg_reg_shift);
13328 %}
13329 
13330 // This pattern is automatically generated from aarch64_ad.m4
13331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13332 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13333 %{
13334   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13335   ins_cost(1.9 * INSN_COST);
13336   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13337 
13338    ins_encode %{
13339      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13340             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13341    %}
13342   ins_pipe(ialu_reg_reg_shift);
13343 %}
13344 
13345 // This pattern is automatically generated from aarch64_ad.m4
13346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13347 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13348 %{
13349   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13350   ins_cost(1.9 * INSN_COST);
13351   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13352 
13353    ins_encode %{
13354      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13355             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13356    %}
13357   ins_pipe(ialu_reg_reg_shift);
13358 %}
13359 
13360 // This pattern is automatically generated from aarch64_ad.m4
13361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13362 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13363 %{
13364   effect(DEF dst, USE src1, USE src2, USE cr);
13365   ins_cost(INSN_COST * 2);
13366   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13367 
13368   ins_encode %{
13369     __ cselw($dst$$Register,
13370              $src1$$Register,
13371              $src2$$Register,
13372              Assembler::LT);
13373   %}
13374   ins_pipe(icond_reg_reg);
13375 %}
13376 
13377 // This pattern is automatically generated from aarch64_ad.m4
13378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13379 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13380 %{
13381   effect(DEF dst, USE src1, USE src2, USE cr);
13382   ins_cost(INSN_COST * 2);
13383   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13384 
13385   ins_encode %{
13386     __ cselw($dst$$Register,
13387              $src1$$Register,
13388              $src2$$Register,
13389              Assembler::GT);
13390   %}
13391   ins_pipe(icond_reg_reg);
13392 %}
13393 
13394 // This pattern is automatically generated from aarch64_ad.m4
13395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13396 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13397 %{
13398   effect(DEF dst, USE src1, USE cr);
13399   ins_cost(INSN_COST * 2);
13400   format %{ "cselw $dst, $src1, zr lt\t"  %}
13401 
13402   ins_encode %{
13403     __ cselw($dst$$Register,
13404              $src1$$Register,
13405              zr,
13406              Assembler::LT);
13407   %}
13408   ins_pipe(icond_reg);
13409 %}
13410 
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13414 %{
13415   effect(DEF dst, USE src1, USE cr);
13416   ins_cost(INSN_COST * 2);
13417   format %{ "cselw $dst, $src1, zr gt\t"  %}
13418 
13419   ins_encode %{
13420     __ cselw($dst$$Register,
13421              $src1$$Register,
13422              zr,
13423              Assembler::GT);
13424   %}
13425   ins_pipe(icond_reg);
13426 %}
13427 
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13431 %{
13432   effect(DEF dst, USE src1, USE cr);
13433   ins_cost(INSN_COST * 2);
13434   format %{ "csincw $dst, $src1, zr le\t"  %}
13435 
13436   ins_encode %{
13437     __ csincw($dst$$Register,
13438              $src1$$Register,
13439              zr,
13440              Assembler::LE);
13441   %}
13442   ins_pipe(icond_reg);
13443 %}
13444 
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13448 %{
13449   effect(DEF dst, USE src1, USE cr);
13450   ins_cost(INSN_COST * 2);
13451   format %{ "csincw $dst, $src1, zr gt\t"  %}
13452 
13453   ins_encode %{
13454     __ csincw($dst$$Register,
13455              $src1$$Register,
13456              zr,
13457              Assembler::GT);
13458   %}
13459   ins_pipe(icond_reg);
13460 %}
13461 
13462 // This pattern is automatically generated from aarch64_ad.m4
13463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13464 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13465 %{
13466   effect(DEF dst, USE src1, USE cr);
13467   ins_cost(INSN_COST * 2);
13468   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13469 
13470   ins_encode %{
13471     __ csinvw($dst$$Register,
13472              $src1$$Register,
13473              zr,
13474              Assembler::LT);
13475   %}
13476   ins_pipe(icond_reg);
13477 %}
13478 
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13482 %{
13483   effect(DEF dst, USE src1, USE cr);
13484   ins_cost(INSN_COST * 2);
13485   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13486 
13487   ins_encode %{
13488     __ csinvw($dst$$Register,
13489              $src1$$Register,
13490              zr,
13491              Assembler::GE);
13492   %}
13493   ins_pipe(icond_reg);
13494 %}
13495 
13496 // This pattern is automatically generated from aarch64_ad.m4
13497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13498 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13499 %{
13500   match(Set dst (MinI src imm));
13501   ins_cost(INSN_COST * 3);
13502   expand %{
13503     rFlagsReg cr;
13504     compI_reg_imm0(cr, src);
13505     cmovI_reg_imm0_lt(dst, src, cr);
13506   %}
13507 %}
13508 
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13512 %{
13513   match(Set dst (MinI imm src));
13514   ins_cost(INSN_COST * 3);
13515   expand %{
13516     rFlagsReg cr;
13517     compI_reg_imm0(cr, src);
13518     cmovI_reg_imm0_lt(dst, src, cr);
13519   %}
13520 %}
13521 
13522 // This pattern is automatically generated from aarch64_ad.m4
13523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13524 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13525 %{
13526   match(Set dst (MinI src imm));
13527   ins_cost(INSN_COST * 3);
13528   expand %{
13529     rFlagsReg cr;
13530     compI_reg_imm0(cr, src);
13531     cmovI_reg_imm1_le(dst, src, cr);
13532   %}
13533 %}
13534 
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13538 %{
13539   match(Set dst (MinI imm src));
13540   ins_cost(INSN_COST * 3);
13541   expand %{
13542     rFlagsReg cr;
13543     compI_reg_imm0(cr, src);
13544     cmovI_reg_imm1_le(dst, src, cr);
13545   %}
13546 %}
13547 
13548 // This pattern is automatically generated from aarch64_ad.m4
13549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13550 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13551 %{
13552   match(Set dst (MinI src imm));
13553   ins_cost(INSN_COST * 3);
13554   expand %{
13555     rFlagsReg cr;
13556     compI_reg_imm0(cr, src);
13557     cmovI_reg_immM1_lt(dst, src, cr);
13558   %}
13559 %}
13560 
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13564 %{
13565   match(Set dst (MinI imm src));
13566   ins_cost(INSN_COST * 3);
13567   expand %{
13568     rFlagsReg cr;
13569     compI_reg_imm0(cr, src);
13570     cmovI_reg_immM1_lt(dst, src, cr);
13571   %}
13572 %}
13573 
13574 // This pattern is automatically generated from aarch64_ad.m4
13575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13576 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13577 %{
13578   match(Set dst (MaxI src imm));
13579   ins_cost(INSN_COST * 3);
13580   expand %{
13581     rFlagsReg cr;
13582     compI_reg_imm0(cr, src);
13583     cmovI_reg_imm0_gt(dst, src, cr);
13584   %}
13585 %}
13586 
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13590 %{
13591   match(Set dst (MaxI imm src));
13592   ins_cost(INSN_COST * 3);
13593   expand %{
13594     rFlagsReg cr;
13595     compI_reg_imm0(cr, src);
13596     cmovI_reg_imm0_gt(dst, src, cr);
13597   %}
13598 %}
13599 
13600 // This pattern is automatically generated from aarch64_ad.m4
13601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13602 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13603 %{
13604   match(Set dst (MaxI src imm));
13605   ins_cost(INSN_COST * 3);
13606   expand %{
13607     rFlagsReg cr;
13608     compI_reg_imm0(cr, src);
13609     cmovI_reg_imm1_gt(dst, src, cr);
13610   %}
13611 %}
13612 
13613 // This pattern is automatically generated from aarch64_ad.m4
13614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13615 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13616 %{
13617   match(Set dst (MaxI imm src));
13618   ins_cost(INSN_COST * 3);
13619   expand %{
13620     rFlagsReg cr;
13621     compI_reg_imm0(cr, src);
13622     cmovI_reg_imm1_gt(dst, src, cr);
13623   %}
13624 %}
13625 
13626 // This pattern is automatically generated from aarch64_ad.m4
13627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13628 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13629 %{
13630   match(Set dst (MaxI src imm));
13631   ins_cost(INSN_COST * 3);
13632   expand %{
13633     rFlagsReg cr;
13634     compI_reg_imm0(cr, src);
13635     cmovI_reg_immM1_ge(dst, src, cr);
13636   %}
13637 %}
13638 
13639 // This pattern is automatically generated from aarch64_ad.m4
13640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13641 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13642 %{
13643   match(Set dst (MaxI imm src));
13644   ins_cost(INSN_COST * 3);
13645   expand %{
13646     rFlagsReg cr;
13647     compI_reg_imm0(cr, src);
13648     cmovI_reg_immM1_ge(dst, src, cr);
13649   %}
13650 %}
13651 
13652 // This pattern is automatically generated from aarch64_ad.m4
13653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13654 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13655 %{
13656   match(Set dst (ReverseI src));
13657   ins_cost(INSN_COST);
13658   format %{ "rbitw  $dst, $src" %}
13659   ins_encode %{
13660     __ rbitw($dst$$Register, $src$$Register);
13661   %}
13662   ins_pipe(ialu_reg);
13663 %}
13664 
13665 // This pattern is automatically generated from aarch64_ad.m4
13666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13667 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13668 %{
13669   match(Set dst (ReverseL src));
13670   ins_cost(INSN_COST);
13671   format %{ "rbit  $dst, $src" %}
13672   ins_encode %{
13673     __ rbit($dst$$Register, $src$$Register);
13674   %}
13675   ins_pipe(ialu_reg);
13676 %}
13677 
13678 
13679 // END This section of the file is automatically generated. Do not edit --------------
13680 
13681 
13682 // ============================================================================
13683 // Floating Point Arithmetic Instructions
13684 
13685 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13686   match(Set dst (AddHF src1 src2));
13687   format %{ "faddh $dst, $src1, $src2" %}
13688   ins_encode %{
13689     __ faddh($dst$$FloatRegister,
13690              $src1$$FloatRegister,
13691              $src2$$FloatRegister);
13692   %}
13693   ins_pipe(fp_dop_reg_reg_s);
13694 %}
13695 
13696 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13697   match(Set dst (AddF src1 src2));
13698 
13699   ins_cost(INSN_COST * 5);
13700   format %{ "fadds   $dst, $src1, $src2" %}
13701 
13702   ins_encode %{
13703     __ fadds(as_FloatRegister($dst$$reg),
13704              as_FloatRegister($src1$$reg),
13705              as_FloatRegister($src2$$reg));
13706   %}
13707 
13708   ins_pipe(fp_dop_reg_reg_s);
13709 %}
13710 
13711 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13712   match(Set dst (AddD src1 src2));
13713 
13714   ins_cost(INSN_COST * 5);
13715   format %{ "faddd   $dst, $src1, $src2" %}
13716 
13717   ins_encode %{
13718     __ faddd(as_FloatRegister($dst$$reg),
13719              as_FloatRegister($src1$$reg),
13720              as_FloatRegister($src2$$reg));
13721   %}
13722 
13723   ins_pipe(fp_dop_reg_reg_d);
13724 %}
13725 
13726 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13727   match(Set dst (SubHF src1 src2));
13728   format %{ "fsubh $dst, $src1, $src2" %}
13729   ins_encode %{
13730     __ fsubh($dst$$FloatRegister,
13731              $src1$$FloatRegister,
13732              $src2$$FloatRegister);
13733   %}
13734   ins_pipe(fp_dop_reg_reg_s);
13735 %}
13736 
13737 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13738   match(Set dst (SubF src1 src2));
13739 
13740   ins_cost(INSN_COST * 5);
13741   format %{ "fsubs   $dst, $src1, $src2" %}
13742 
13743   ins_encode %{
13744     __ fsubs(as_FloatRegister($dst$$reg),
13745              as_FloatRegister($src1$$reg),
13746              as_FloatRegister($src2$$reg));
13747   %}
13748 
13749   ins_pipe(fp_dop_reg_reg_s);
13750 %}
13751 
13752 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13753   match(Set dst (SubD src1 src2));
13754 
13755   ins_cost(INSN_COST * 5);
13756   format %{ "fsubd   $dst, $src1, $src2" %}
13757 
13758   ins_encode %{
13759     __ fsubd(as_FloatRegister($dst$$reg),
13760              as_FloatRegister($src1$$reg),
13761              as_FloatRegister($src2$$reg));
13762   %}
13763 
13764   ins_pipe(fp_dop_reg_reg_d);
13765 %}
13766 
13767 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13768   match(Set dst (MulHF src1 src2));
13769   format %{ "fmulh $dst, $src1, $src2" %}
13770   ins_encode %{
13771     __ fmulh($dst$$FloatRegister,
13772              $src1$$FloatRegister,
13773              $src2$$FloatRegister);
13774   %}
13775   ins_pipe(fp_dop_reg_reg_s);
13776 %}
13777 
13778 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13779   match(Set dst (MulF src1 src2));
13780 
13781   ins_cost(INSN_COST * 6);
13782   format %{ "fmuls   $dst, $src1, $src2" %}
13783 
13784   ins_encode %{
13785     __ fmuls(as_FloatRegister($dst$$reg),
13786              as_FloatRegister($src1$$reg),
13787              as_FloatRegister($src2$$reg));
13788   %}
13789 
13790   ins_pipe(fp_dop_reg_reg_s);
13791 %}
13792 
13793 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13794   match(Set dst (MulD src1 src2));
13795 
13796   ins_cost(INSN_COST * 6);
13797   format %{ "fmuld   $dst, $src1, $src2" %}
13798 
13799   ins_encode %{
13800     __ fmuld(as_FloatRegister($dst$$reg),
13801              as_FloatRegister($src1$$reg),
13802              as_FloatRegister($src2$$reg));
13803   %}
13804 
13805   ins_pipe(fp_dop_reg_reg_d);
13806 %}
13807 
13808 // src1 * src2 + src3 (half-precision float)
13809 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13810   match(Set dst (FmaHF src3 (Binary src1 src2)));
13811   format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13812   ins_encode %{
13813     assert(UseFMA, "Needs FMA instructions support.");
13814     __ fmaddh($dst$$FloatRegister,
13815               $src1$$FloatRegister,
13816               $src2$$FloatRegister,
13817               $src3$$FloatRegister);
13818   %}
13819   ins_pipe(pipe_class_default);
13820 %}
13821 
13822 // src1 * src2 + src3
13823 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13824   match(Set dst (FmaF src3 (Binary src1 src2)));
13825 
13826   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13827 
13828   ins_encode %{
13829     assert(UseFMA, "Needs FMA instructions support.");
13830     __ fmadds(as_FloatRegister($dst$$reg),
13831              as_FloatRegister($src1$$reg),
13832              as_FloatRegister($src2$$reg),
13833              as_FloatRegister($src3$$reg));
13834   %}
13835 
13836   ins_pipe(pipe_class_default);
13837 %}
13838 
13839 // src1 * src2 + src3
13840 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13841   match(Set dst (FmaD src3 (Binary src1 src2)));
13842 
13843   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13844 
13845   ins_encode %{
13846     assert(UseFMA, "Needs FMA instructions support.");
13847     __ fmaddd(as_FloatRegister($dst$$reg),
13848              as_FloatRegister($src1$$reg),
13849              as_FloatRegister($src2$$reg),
13850              as_FloatRegister($src3$$reg));
13851   %}
13852 
13853   ins_pipe(pipe_class_default);
13854 %}
13855 
13856 // src1 * (-src2) + src3
13857 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13858 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13859   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13860 
13861   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13862 
13863   ins_encode %{
13864     assert(UseFMA, "Needs FMA instructions support.");
13865     __ fmsubs(as_FloatRegister($dst$$reg),
13866               as_FloatRegister($src1$$reg),
13867               as_FloatRegister($src2$$reg),
13868               as_FloatRegister($src3$$reg));
13869   %}
13870 
13871   ins_pipe(pipe_class_default);
13872 %}
13873 
13874 // src1 * (-src2) + src3
13875 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13876 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13877   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13878 
13879   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13880 
13881   ins_encode %{
13882     assert(UseFMA, "Needs FMA instructions support.");
13883     __ fmsubd(as_FloatRegister($dst$$reg),
13884               as_FloatRegister($src1$$reg),
13885               as_FloatRegister($src2$$reg),
13886               as_FloatRegister($src3$$reg));
13887   %}
13888 
13889   ins_pipe(pipe_class_default);
13890 %}
13891 
13892 // src1 * (-src2) - src3
13893 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13894 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13895   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13896 
13897   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13898 
13899   ins_encode %{
13900     assert(UseFMA, "Needs FMA instructions support.");
13901     __ fnmadds(as_FloatRegister($dst$$reg),
13902                as_FloatRegister($src1$$reg),
13903                as_FloatRegister($src2$$reg),
13904                as_FloatRegister($src3$$reg));
13905   %}
13906 
13907   ins_pipe(pipe_class_default);
13908 %}
13909 
13910 // src1 * (-src2) - src3
13911 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13912 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13913   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13914 
13915   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13916 
13917   ins_encode %{
13918     assert(UseFMA, "Needs FMA instructions support.");
13919     __ fnmaddd(as_FloatRegister($dst$$reg),
13920                as_FloatRegister($src1$$reg),
13921                as_FloatRegister($src2$$reg),
13922                as_FloatRegister($src3$$reg));
13923   %}
13924 
13925   ins_pipe(pipe_class_default);
13926 %}
13927 
13928 // src1 * src2 - src3
13929 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13930   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13931 
13932   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13933 
13934   ins_encode %{
13935     assert(UseFMA, "Needs FMA instructions support.");
13936     __ fnmsubs(as_FloatRegister($dst$$reg),
13937                as_FloatRegister($src1$$reg),
13938                as_FloatRegister($src2$$reg),
13939                as_FloatRegister($src3$$reg));
13940   %}
13941 
13942   ins_pipe(pipe_class_default);
13943 %}
13944 
13945 // src1 * src2 - src3
13946 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13947   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13948 
13949   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13950 
13951   ins_encode %{
13952     assert(UseFMA, "Needs FMA instructions support.");
13953     // n.b. insn name should be fnmsubd
13954     __ fnmsub(as_FloatRegister($dst$$reg),
13955               as_FloatRegister($src1$$reg),
13956               as_FloatRegister($src2$$reg),
13957               as_FloatRegister($src3$$reg));
13958   %}
13959 
13960   ins_pipe(pipe_class_default);
13961 %}
13962 
13963 // Math.max(HH)H (half-precision float)
13964 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13965   match(Set dst (MaxHF src1 src2));
13966   format %{ "fmaxh $dst, $src1, $src2" %}
13967   ins_encode %{
13968     __ fmaxh($dst$$FloatRegister,
13969              $src1$$FloatRegister,
13970              $src2$$FloatRegister);
13971   %}
13972   ins_pipe(fp_dop_reg_reg_s);
13973 %}
13974 
13975 // Math.min(HH)H (half-precision float)
13976 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13977   match(Set dst (MinHF src1 src2));
13978   format %{ "fminh $dst, $src1, $src2" %}
13979   ins_encode %{
13980     __ fminh($dst$$FloatRegister,
13981              $src1$$FloatRegister,
13982              $src2$$FloatRegister);
13983   %}
13984   ins_pipe(fp_dop_reg_reg_s);
13985 %}
13986 
13987 // Math.max(FF)F
13988 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13989   match(Set dst (MaxF src1 src2));
13990 
13991   format %{ "fmaxs   $dst, $src1, $src2" %}
13992   ins_encode %{
13993     __ fmaxs(as_FloatRegister($dst$$reg),
13994              as_FloatRegister($src1$$reg),
13995              as_FloatRegister($src2$$reg));
13996   %}
13997 
13998   ins_pipe(fp_dop_reg_reg_s);
13999 %}
14000 
14001 // Math.min(FF)F
14002 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14003   match(Set dst (MinF src1 src2));
14004 
14005   format %{ "fmins   $dst, $src1, $src2" %}
14006   ins_encode %{
14007     __ fmins(as_FloatRegister($dst$$reg),
14008              as_FloatRegister($src1$$reg),
14009              as_FloatRegister($src2$$reg));
14010   %}
14011 
14012   ins_pipe(fp_dop_reg_reg_s);
14013 %}
14014 
14015 // Math.max(DD)D
14016 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14017   match(Set dst (MaxD src1 src2));
14018 
14019   format %{ "fmaxd   $dst, $src1, $src2" %}
14020   ins_encode %{
14021     __ fmaxd(as_FloatRegister($dst$$reg),
14022              as_FloatRegister($src1$$reg),
14023              as_FloatRegister($src2$$reg));
14024   %}
14025 
14026   ins_pipe(fp_dop_reg_reg_d);
14027 %}
14028 
14029 // Math.min(DD)D
14030 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14031   match(Set dst (MinD src1 src2));
14032 
14033   format %{ "fmind   $dst, $src1, $src2" %}
14034   ins_encode %{
14035     __ fmind(as_FloatRegister($dst$$reg),
14036              as_FloatRegister($src1$$reg),
14037              as_FloatRegister($src2$$reg));
14038   %}
14039 
14040   ins_pipe(fp_dop_reg_reg_d);
14041 %}
14042 
14043 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14044   match(Set dst (DivHF src1  src2));
14045   format %{ "fdivh $dst, $src1, $src2" %}
14046   ins_encode %{
14047     __ fdivh($dst$$FloatRegister,
14048              $src1$$FloatRegister,
14049              $src2$$FloatRegister);
14050   %}
14051   ins_pipe(fp_div_s);
14052 %}
14053 
14054 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14055   match(Set dst (DivF src1  src2));
14056 
14057   ins_cost(INSN_COST * 18);
14058   format %{ "fdivs   $dst, $src1, $src2" %}
14059 
14060   ins_encode %{
14061     __ fdivs(as_FloatRegister($dst$$reg),
14062              as_FloatRegister($src1$$reg),
14063              as_FloatRegister($src2$$reg));
14064   %}
14065 
14066   ins_pipe(fp_div_s);
14067 %}
14068 
14069 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14070   match(Set dst (DivD src1  src2));
14071 
14072   ins_cost(INSN_COST * 32);
14073   format %{ "fdivd   $dst, $src1, $src2" %}
14074 
14075   ins_encode %{
14076     __ fdivd(as_FloatRegister($dst$$reg),
14077              as_FloatRegister($src1$$reg),
14078              as_FloatRegister($src2$$reg));
14079   %}
14080 
14081   ins_pipe(fp_div_d);
14082 %}
14083 
14084 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14085   match(Set dst (NegF src));
14086 
14087   ins_cost(INSN_COST * 3);
14088   format %{ "fneg   $dst, $src" %}
14089 
14090   ins_encode %{
14091     __ fnegs(as_FloatRegister($dst$$reg),
14092              as_FloatRegister($src$$reg));
14093   %}
14094 
14095   ins_pipe(fp_uop_s);
14096 %}
14097 
14098 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14099   match(Set dst (NegD src));
14100 
14101   ins_cost(INSN_COST * 3);
14102   format %{ "fnegd   $dst, $src" %}
14103 
14104   ins_encode %{
14105     __ fnegd(as_FloatRegister($dst$$reg),
14106              as_FloatRegister($src$$reg));
14107   %}
14108 
14109   ins_pipe(fp_uop_d);
14110 %}
14111 
14112 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14113 %{
14114   match(Set dst (AbsI src));
14115 
14116   effect(KILL cr);
14117   ins_cost(INSN_COST * 2);
14118   format %{ "cmpw  $src, zr\n\t"
14119             "cnegw $dst, $src, Assembler::LT\t# int abs"
14120   %}
14121 
14122   ins_encode %{
14123     __ cmpw(as_Register($src$$reg), zr);
14124     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14125   %}
14126   ins_pipe(pipe_class_default);
14127 %}
14128 
14129 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14130 %{
14131   match(Set dst (AbsL src));
14132 
14133   effect(KILL cr);
14134   ins_cost(INSN_COST * 2);
14135   format %{ "cmp  $src, zr\n\t"
14136             "cneg $dst, $src, Assembler::LT\t# long abs"
14137   %}
14138 
14139   ins_encode %{
14140     __ cmp(as_Register($src$$reg), zr);
14141     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14142   %}
14143   ins_pipe(pipe_class_default);
14144 %}
14145 
14146 instruct absF_reg(vRegF dst, vRegF src) %{
14147   match(Set dst (AbsF src));
14148 
14149   ins_cost(INSN_COST * 3);
14150   format %{ "fabss   $dst, $src" %}
14151   ins_encode %{
14152     __ fabss(as_FloatRegister($dst$$reg),
14153              as_FloatRegister($src$$reg));
14154   %}
14155 
14156   ins_pipe(fp_uop_s);
14157 %}
14158 
14159 instruct absD_reg(vRegD dst, vRegD src) %{
14160   match(Set dst (AbsD src));
14161 
14162   ins_cost(INSN_COST * 3);
14163   format %{ "fabsd   $dst, $src" %}
14164   ins_encode %{
14165     __ fabsd(as_FloatRegister($dst$$reg),
14166              as_FloatRegister($src$$reg));
14167   %}
14168 
14169   ins_pipe(fp_uop_d);
14170 %}
14171 
14172 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14173   match(Set dst (AbsF (SubF src1 src2)));
14174 
14175   ins_cost(INSN_COST * 3);
14176   format %{ "fabds   $dst, $src1, $src2" %}
14177   ins_encode %{
14178     __ fabds(as_FloatRegister($dst$$reg),
14179              as_FloatRegister($src1$$reg),
14180              as_FloatRegister($src2$$reg));
14181   %}
14182 
14183   ins_pipe(fp_uop_s);
14184 %}
14185 
14186 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14187   match(Set dst (AbsD (SubD src1 src2)));
14188 
14189   ins_cost(INSN_COST * 3);
14190   format %{ "fabdd   $dst, $src1, $src2" %}
14191   ins_encode %{
14192     __ fabdd(as_FloatRegister($dst$$reg),
14193              as_FloatRegister($src1$$reg),
14194              as_FloatRegister($src2$$reg));
14195   %}
14196 
14197   ins_pipe(fp_uop_d);
14198 %}
14199 
14200 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14201   match(Set dst (SqrtD src));
14202 
14203   ins_cost(INSN_COST * 50);
14204   format %{ "fsqrtd  $dst, $src" %}
14205   ins_encode %{
14206     __ fsqrtd(as_FloatRegister($dst$$reg),
14207              as_FloatRegister($src$$reg));
14208   %}
14209 
14210   ins_pipe(fp_div_s);
14211 %}
14212 
14213 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14214   match(Set dst (SqrtF src));
14215 
14216   ins_cost(INSN_COST * 50);
14217   format %{ "fsqrts  $dst, $src" %}
14218   ins_encode %{
14219     __ fsqrts(as_FloatRegister($dst$$reg),
14220              as_FloatRegister($src$$reg));
14221   %}
14222 
14223   ins_pipe(fp_div_d);
14224 %}
14225 
14226 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14227   match(Set dst (SqrtHF src));
14228   format %{ "fsqrth $dst, $src" %}
14229   ins_encode %{
14230     __ fsqrth($dst$$FloatRegister,
14231               $src$$FloatRegister);
14232   %}
14233   ins_pipe(fp_div_s);
14234 %}
14235 
14236 // Math.rint, floor, ceil
14237 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14238   match(Set dst (RoundDoubleMode src rmode));
14239   format %{ "frint  $dst, $src, $rmode" %}
14240   ins_encode %{
14241     switch ($rmode$$constant) {
14242       case RoundDoubleModeNode::rmode_rint:
14243         __ frintnd(as_FloatRegister($dst$$reg),
14244                    as_FloatRegister($src$$reg));
14245         break;
14246       case RoundDoubleModeNode::rmode_floor:
14247         __ frintmd(as_FloatRegister($dst$$reg),
14248                    as_FloatRegister($src$$reg));
14249         break;
14250       case RoundDoubleModeNode::rmode_ceil:
14251         __ frintpd(as_FloatRegister($dst$$reg),
14252                    as_FloatRegister($src$$reg));
14253         break;
14254     }
14255   %}
14256   ins_pipe(fp_uop_d);
14257 %}
14258 
14259 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14260   match(Set dst (CopySignD src1 (Binary src2 zero)));
14261   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14262   format %{ "CopySignD  $dst $src1 $src2" %}
14263   ins_encode %{
14264     FloatRegister dst = as_FloatRegister($dst$$reg),
14265                   src1 = as_FloatRegister($src1$$reg),
14266                   src2 = as_FloatRegister($src2$$reg),
14267                   zero = as_FloatRegister($zero$$reg);
14268     __ fnegd(dst, zero);
14269     __ bsl(dst, __ T8B, src2, src1);
14270   %}
14271   ins_pipe(fp_uop_d);
14272 %}
14273 
14274 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14275   match(Set dst (CopySignF src1 src2));
14276   effect(TEMP_DEF dst, USE src1, USE src2);
14277   format %{ "CopySignF  $dst $src1 $src2" %}
14278   ins_encode %{
14279     FloatRegister dst = as_FloatRegister($dst$$reg),
14280                   src1 = as_FloatRegister($src1$$reg),
14281                   src2 = as_FloatRegister($src2$$reg);
14282     __ movi(dst, __ T2S, 0x80, 24);
14283     __ bsl(dst, __ T8B, src2, src1);
14284   %}
14285   ins_pipe(fp_uop_d);
14286 %}
14287 
14288 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14289   match(Set dst (SignumD src (Binary zero one)));
14290   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14291   format %{ "signumD  $dst, $src" %}
14292   ins_encode %{
14293     FloatRegister src = as_FloatRegister($src$$reg),
14294                   dst = as_FloatRegister($dst$$reg),
14295                   zero = as_FloatRegister($zero$$reg),
14296                   one = as_FloatRegister($one$$reg);
14297     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14298     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14299     // Bit selection instruction gets bit from "one" for each enabled bit in
14300     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14301     // NaN the whole "src" will be copied because "dst" is zero. For all other
14302     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14303     // from "src", and all other bits are copied from 1.0.
14304     __ bsl(dst, __ T8B, one, src);
14305   %}
14306   ins_pipe(fp_uop_d);
14307 %}
14308 
14309 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14310   match(Set dst (SignumF src (Binary zero one)));
14311   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14312   format %{ "signumF  $dst, $src" %}
14313   ins_encode %{
14314     FloatRegister src = as_FloatRegister($src$$reg),
14315                   dst = as_FloatRegister($dst$$reg),
14316                   zero = as_FloatRegister($zero$$reg),
14317                   one = as_FloatRegister($one$$reg);
14318     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14319     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14320     // Bit selection instruction gets bit from "one" for each enabled bit in
14321     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14322     // NaN the whole "src" will be copied because "dst" is zero. For all other
14323     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14324     // from "src", and all other bits are copied from 1.0.
14325     __ bsl(dst, __ T8B, one, src);
14326   %}
14327   ins_pipe(fp_uop_d);
14328 %}
14329 
14330 instruct onspinwait() %{
14331   match(OnSpinWait);
14332   ins_cost(INSN_COST);
14333 
14334   format %{ "onspinwait" %}
14335 
14336   ins_encode %{
14337     __ spin_wait();
14338   %}
14339   ins_pipe(pipe_class_empty);
14340 %}
14341 
14342 // ============================================================================
14343 // Logical Instructions
14344 
14345 // Integer Logical Instructions
14346 
14347 // And Instructions
14348 
14349 
14350 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14351   match(Set dst (AndI src1 src2));
14352 
14353   format %{ "andw  $dst, $src1, $src2\t# int" %}
14354 
14355   ins_cost(INSN_COST);
14356   ins_encode %{
14357     __ andw(as_Register($dst$$reg),
14358             as_Register($src1$$reg),
14359             as_Register($src2$$reg));
14360   %}
14361 
14362   ins_pipe(ialu_reg_reg);
14363 %}
14364 
14365 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14366   match(Set dst (AndI src1 src2));
14367 
14368   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14369 
14370   ins_cost(INSN_COST);
14371   ins_encode %{
14372     __ andw(as_Register($dst$$reg),
14373             as_Register($src1$$reg),
14374             (uint64_t)($src2$$constant));
14375   %}
14376 
14377   ins_pipe(ialu_reg_imm);
14378 %}
14379 
14380 // Or Instructions
14381 
14382 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14383   match(Set dst (OrI src1 src2));
14384 
14385   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14386 
14387   ins_cost(INSN_COST);
14388   ins_encode %{
14389     __ orrw(as_Register($dst$$reg),
14390             as_Register($src1$$reg),
14391             as_Register($src2$$reg));
14392   %}
14393 
14394   ins_pipe(ialu_reg_reg);
14395 %}
14396 
14397 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14398   match(Set dst (OrI src1 src2));
14399 
14400   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14401 
14402   ins_cost(INSN_COST);
14403   ins_encode %{
14404     __ orrw(as_Register($dst$$reg),
14405             as_Register($src1$$reg),
14406             (uint64_t)($src2$$constant));
14407   %}
14408 
14409   ins_pipe(ialu_reg_imm);
14410 %}
14411 
14412 // Xor Instructions
14413 
14414 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14415   match(Set dst (XorI src1 src2));
14416 
14417   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14418 
14419   ins_cost(INSN_COST);
14420   ins_encode %{
14421     __ eorw(as_Register($dst$$reg),
14422             as_Register($src1$$reg),
14423             as_Register($src2$$reg));
14424   %}
14425 
14426   ins_pipe(ialu_reg_reg);
14427 %}
14428 
14429 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14430   match(Set dst (XorI src1 src2));
14431 
14432   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14433 
14434   ins_cost(INSN_COST);
14435   ins_encode %{
14436     __ eorw(as_Register($dst$$reg),
14437             as_Register($src1$$reg),
14438             (uint64_t)($src2$$constant));
14439   %}
14440 
14441   ins_pipe(ialu_reg_imm);
14442 %}
14443 
14444 // Long Logical Instructions
14445 // TODO
14446 
14447 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14448   match(Set dst (AndL src1 src2));
14449 
14450   format %{ "and  $dst, $src1, $src2\t# int" %}
14451 
14452   ins_cost(INSN_COST);
14453   ins_encode %{
14454     __ andr(as_Register($dst$$reg),
14455             as_Register($src1$$reg),
14456             as_Register($src2$$reg));
14457   %}
14458 
14459   ins_pipe(ialu_reg_reg);
14460 %}
14461 
14462 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14463   match(Set dst (AndL src1 src2));
14464 
14465   format %{ "and  $dst, $src1, $src2\t# int" %}
14466 
14467   ins_cost(INSN_COST);
14468   ins_encode %{
14469     __ andr(as_Register($dst$$reg),
14470             as_Register($src1$$reg),
14471             (uint64_t)($src2$$constant));
14472   %}
14473 
14474   ins_pipe(ialu_reg_imm);
14475 %}
14476 
14477 // Or Instructions
14478 
14479 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14480   match(Set dst (OrL src1 src2));
14481 
14482   format %{ "orr  $dst, $src1, $src2\t# int" %}
14483 
14484   ins_cost(INSN_COST);
14485   ins_encode %{
14486     __ orr(as_Register($dst$$reg),
14487            as_Register($src1$$reg),
14488            as_Register($src2$$reg));
14489   %}
14490 
14491   ins_pipe(ialu_reg_reg);
14492 %}
14493 
14494 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14495   match(Set dst (OrL src1 src2));
14496 
14497   format %{ "orr  $dst, $src1, $src2\t# int" %}
14498 
14499   ins_cost(INSN_COST);
14500   ins_encode %{
14501     __ orr(as_Register($dst$$reg),
14502            as_Register($src1$$reg),
14503            (uint64_t)($src2$$constant));
14504   %}
14505 
14506   ins_pipe(ialu_reg_imm);
14507 %}
14508 
14509 // Xor Instructions
14510 
14511 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14512   match(Set dst (XorL src1 src2));
14513 
14514   format %{ "eor  $dst, $src1, $src2\t# int" %}
14515 
14516   ins_cost(INSN_COST);
14517   ins_encode %{
14518     __ eor(as_Register($dst$$reg),
14519            as_Register($src1$$reg),
14520            as_Register($src2$$reg));
14521   %}
14522 
14523   ins_pipe(ialu_reg_reg);
14524 %}
14525 
14526 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14527   match(Set dst (XorL src1 src2));
14528 
14529   ins_cost(INSN_COST);
14530   format %{ "eor  $dst, $src1, $src2\t# int" %}
14531 
14532   ins_encode %{
14533     __ eor(as_Register($dst$$reg),
14534            as_Register($src1$$reg),
14535            (uint64_t)($src2$$constant));
14536   %}
14537 
14538   ins_pipe(ialu_reg_imm);
14539 %}
14540 
14541 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14542 %{
14543   match(Set dst (ConvI2L src));
14544 
14545   ins_cost(INSN_COST);
14546   format %{ "sxtw  $dst, $src\t# i2l" %}
14547   ins_encode %{
14548     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14549   %}
14550   ins_pipe(ialu_reg_shift);
14551 %}
14552 
14553 // this pattern occurs in bigmath arithmetic
14554 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14555 %{
14556   match(Set dst (AndL (ConvI2L src) mask));
14557 
14558   ins_cost(INSN_COST);
14559   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14560   ins_encode %{
14561     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14562   %}
14563 
14564   ins_pipe(ialu_reg_shift);
14565 %}
14566 
14567 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14568   match(Set dst (ConvL2I src));
14569 
14570   ins_cost(INSN_COST);
14571   format %{ "movw  $dst, $src \t// l2i" %}
14572 
14573   ins_encode %{
14574     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14575   %}
14576 
14577   ins_pipe(ialu_reg);
14578 %}
14579 
14580 instruct convD2F_reg(vRegF dst, vRegD src) %{
14581   match(Set dst (ConvD2F src));
14582 
14583   ins_cost(INSN_COST * 5);
14584   format %{ "fcvtd  $dst, $src \t// d2f" %}
14585 
14586   ins_encode %{
14587     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14588   %}
14589 
14590   ins_pipe(fp_d2f);
14591 %}
14592 
14593 instruct convF2D_reg(vRegD dst, vRegF src) %{
14594   match(Set dst (ConvF2D src));
14595 
14596   ins_cost(INSN_COST * 5);
14597   format %{ "fcvts  $dst, $src \t// f2d" %}
14598 
14599   ins_encode %{
14600     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14601   %}
14602 
14603   ins_pipe(fp_f2d);
14604 %}
14605 
14606 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14607   match(Set dst (ConvF2I src));
14608 
14609   ins_cost(INSN_COST * 5);
14610   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14611 
14612   ins_encode %{
14613     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14614   %}
14615 
14616   ins_pipe(fp_f2i);
14617 %}
14618 
14619 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14620   match(Set dst (ConvF2L src));
14621 
14622   ins_cost(INSN_COST * 5);
14623   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14624 
14625   ins_encode %{
14626     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14627   %}
14628 
14629   ins_pipe(fp_f2l);
14630 %}
14631 
14632 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14633   match(Set dst (ConvF2HF src));
14634   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14635             "smov $dst, $tmp\t# move result from $tmp to $dst"
14636   %}
14637   effect(TEMP tmp);
14638   ins_encode %{
14639       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14640   %}
14641   ins_pipe(pipe_slow);
14642 %}
14643 
14644 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14645   match(Set dst (ConvHF2F src));
14646   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14647             "fcvt $dst, $tmp\t# convert half to single precision"
14648   %}
14649   effect(TEMP tmp);
14650   ins_encode %{
14651       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14652   %}
14653   ins_pipe(pipe_slow);
14654 %}
14655 
14656 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14657   match(Set dst (ConvI2F src));
14658 
14659   ins_cost(INSN_COST * 5);
14660   format %{ "scvtfws  $dst, $src \t// i2f" %}
14661 
14662   ins_encode %{
14663     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14664   %}
14665 
14666   ins_pipe(fp_i2f);
14667 %}
14668 
14669 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14670   match(Set dst (ConvL2F src));
14671 
14672   ins_cost(INSN_COST * 5);
14673   format %{ "scvtfs  $dst, $src \t// l2f" %}
14674 
14675   ins_encode %{
14676     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14677   %}
14678 
14679   ins_pipe(fp_l2f);
14680 %}
14681 
14682 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14683   match(Set dst (ConvD2I src));
14684 
14685   ins_cost(INSN_COST * 5);
14686   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14687 
14688   ins_encode %{
14689     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14690   %}
14691 
14692   ins_pipe(fp_d2i);
14693 %}
14694 
14695 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14696   match(Set dst (ConvD2L src));
14697 
14698   ins_cost(INSN_COST * 5);
14699   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14700 
14701   ins_encode %{
14702     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14703   %}
14704 
14705   ins_pipe(fp_d2l);
14706 %}
14707 
14708 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14709   match(Set dst (ConvI2D src));
14710 
14711   ins_cost(INSN_COST * 5);
14712   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14713 
14714   ins_encode %{
14715     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14716   %}
14717 
14718   ins_pipe(fp_i2d);
14719 %}
14720 
14721 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14722   match(Set dst (ConvL2D src));
14723 
14724   ins_cost(INSN_COST * 5);
14725   format %{ "scvtfd  $dst, $src \t// l2d" %}
14726 
14727   ins_encode %{
14728     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14729   %}
14730 
14731   ins_pipe(fp_l2d);
14732 %}
14733 
14734 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14735 %{
14736   match(Set dst (RoundD src));
14737   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14738   format %{ "java_round_double $dst,$src"%}
14739   ins_encode %{
14740     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14741                          as_FloatRegister($ftmp$$reg));
14742   %}
14743   ins_pipe(pipe_slow);
14744 %}
14745 
14746 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14747 %{
14748   match(Set dst (RoundF src));
14749   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14750   format %{ "java_round_float $dst,$src"%}
14751   ins_encode %{
14752     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14753                         as_FloatRegister($ftmp$$reg));
14754   %}
14755   ins_pipe(pipe_slow);
14756 %}
14757 
14758 // stack <-> reg and reg <-> reg shuffles with no conversion
14759 
14760 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14761 
14762   match(Set dst (MoveF2I src));
14763 
14764   effect(DEF dst, USE src);
14765 
14766   ins_cost(4 * INSN_COST);
14767 
14768   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14769 
14770   ins_encode %{
14771     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14772   %}
14773 
14774   ins_pipe(iload_reg_reg);
14775 
14776 %}
14777 
14778 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14779 
14780   match(Set dst (MoveI2F src));
14781 
14782   effect(DEF dst, USE src);
14783 
14784   ins_cost(4 * INSN_COST);
14785 
14786   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14787 
14788   ins_encode %{
14789     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14790   %}
14791 
14792   ins_pipe(pipe_class_memory);
14793 
14794 %}
14795 
14796 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14797 
14798   match(Set dst (MoveD2L src));
14799 
14800   effect(DEF dst, USE src);
14801 
14802   ins_cost(4 * INSN_COST);
14803 
14804   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14805 
14806   ins_encode %{
14807     __ ldr($dst$$Register, Address(sp, $src$$disp));
14808   %}
14809 
14810   ins_pipe(iload_reg_reg);
14811 
14812 %}
14813 
14814 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14815 
14816   match(Set dst (MoveL2D src));
14817 
14818   effect(DEF dst, USE src);
14819 
14820   ins_cost(4 * INSN_COST);
14821 
14822   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14823 
14824   ins_encode %{
14825     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14826   %}
14827 
14828   ins_pipe(pipe_class_memory);
14829 
14830 %}
14831 
14832 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14833 
14834   match(Set dst (MoveF2I src));
14835 
14836   effect(DEF dst, USE src);
14837 
14838   ins_cost(INSN_COST);
14839 
14840   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14841 
14842   ins_encode %{
14843     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14844   %}
14845 
14846   ins_pipe(pipe_class_memory);
14847 
14848 %}
14849 
14850 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14851 
14852   match(Set dst (MoveI2F src));
14853 
14854   effect(DEF dst, USE src);
14855 
14856   ins_cost(INSN_COST);
14857 
14858   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14859 
14860   ins_encode %{
14861     __ strw($src$$Register, Address(sp, $dst$$disp));
14862   %}
14863 
14864   ins_pipe(istore_reg_reg);
14865 
14866 %}
14867 
14868 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14869 
14870   match(Set dst (MoveD2L src));
14871 
14872   effect(DEF dst, USE src);
14873 
14874   ins_cost(INSN_COST);
14875 
14876   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14877 
14878   ins_encode %{
14879     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14880   %}
14881 
14882   ins_pipe(pipe_class_memory);
14883 
14884 %}
14885 
14886 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14887 
14888   match(Set dst (MoveL2D src));
14889 
14890   effect(DEF dst, USE src);
14891 
14892   ins_cost(INSN_COST);
14893 
14894   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14895 
14896   ins_encode %{
14897     __ str($src$$Register, Address(sp, $dst$$disp));
14898   %}
14899 
14900   ins_pipe(istore_reg_reg);
14901 
14902 %}
14903 
14904 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14905 
14906   match(Set dst (MoveF2I src));
14907 
14908   effect(DEF dst, USE src);
14909 
14910   ins_cost(INSN_COST);
14911 
14912   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14913 
14914   ins_encode %{
14915     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14916   %}
14917 
14918   ins_pipe(fp_f2i);
14919 
14920 %}
14921 
14922 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14923 
14924   match(Set dst (MoveI2F src));
14925 
14926   effect(DEF dst, USE src);
14927 
14928   ins_cost(INSN_COST);
14929 
14930   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14931 
14932   ins_encode %{
14933     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14934   %}
14935 
14936   ins_pipe(fp_i2f);
14937 
14938 %}
14939 
14940 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14941 
14942   match(Set dst (MoveD2L src));
14943 
14944   effect(DEF dst, USE src);
14945 
14946   ins_cost(INSN_COST);
14947 
14948   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14949 
14950   ins_encode %{
14951     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14952   %}
14953 
14954   ins_pipe(fp_d2l);
14955 
14956 %}
14957 
14958 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14959 
14960   match(Set dst (MoveL2D src));
14961 
14962   effect(DEF dst, USE src);
14963 
14964   ins_cost(INSN_COST);
14965 
14966   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14967 
14968   ins_encode %{
14969     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14970   %}
14971 
14972   ins_pipe(fp_l2d);
14973 
14974 %}
14975 
14976 // ============================================================================
14977 // clearing of an array
14978 
14979 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14980 %{
14981   match(Set dummy (ClearArray cnt base));
14982   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14983 
14984   ins_cost(4 * INSN_COST);
14985   format %{ "ClearArray $cnt, $base" %}
14986 
14987   ins_encode %{
14988     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14989     if (tpc == nullptr) {
14990       ciEnv::current()->record_failure("CodeCache is full");
14991       return;
14992     }
14993   %}
14994 
14995   ins_pipe(pipe_class_memory);
14996 %}
14997 
14998 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14999 %{
15000   predicate((uint64_t)n->in(2)->get_long()
15001             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15002   match(Set dummy (ClearArray cnt base));
15003   effect(TEMP temp, USE_KILL base, KILL cr);
15004 
15005   ins_cost(4 * INSN_COST);
15006   format %{ "ClearArray $cnt, $base" %}
15007 
15008   ins_encode %{
15009     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15010     if (tpc == nullptr) {
15011       ciEnv::current()->record_failure("CodeCache is full");
15012       return;
15013     }
15014   %}
15015 
15016   ins_pipe(pipe_class_memory);
15017 %}
15018 
15019 // ============================================================================
15020 // Overflow Math Instructions
15021 
15022 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15023 %{
15024   match(Set cr (OverflowAddI op1 op2));
15025 
15026   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15027   ins_cost(INSN_COST);
15028   ins_encode %{
15029     __ cmnw($op1$$Register, $op2$$Register);
15030   %}
15031 
15032   ins_pipe(icmp_reg_reg);
15033 %}
15034 
15035 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15036 %{
15037   match(Set cr (OverflowAddI op1 op2));
15038 
15039   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15040   ins_cost(INSN_COST);
15041   ins_encode %{
15042     __ cmnw($op1$$Register, $op2$$constant);
15043   %}
15044 
15045   ins_pipe(icmp_reg_imm);
15046 %}
15047 
15048 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15049 %{
15050   match(Set cr (OverflowAddL op1 op2));
15051 
15052   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15053   ins_cost(INSN_COST);
15054   ins_encode %{
15055     __ cmn($op1$$Register, $op2$$Register);
15056   %}
15057 
15058   ins_pipe(icmp_reg_reg);
15059 %}
15060 
15061 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15062 %{
15063   match(Set cr (OverflowAddL op1 op2));
15064 
15065   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15066   ins_cost(INSN_COST);
15067   ins_encode %{
15068     __ adds(zr, $op1$$Register, $op2$$constant);
15069   %}
15070 
15071   ins_pipe(icmp_reg_imm);
15072 %}
15073 
15074 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15075 %{
15076   match(Set cr (OverflowSubI op1 op2));
15077 
15078   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15079   ins_cost(INSN_COST);
15080   ins_encode %{
15081     __ cmpw($op1$$Register, $op2$$Register);
15082   %}
15083 
15084   ins_pipe(icmp_reg_reg);
15085 %}
15086 
15087 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15088 %{
15089   match(Set cr (OverflowSubI op1 op2));
15090 
15091   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15092   ins_cost(INSN_COST);
15093   ins_encode %{
15094     __ cmpw($op1$$Register, $op2$$constant);
15095   %}
15096 
15097   ins_pipe(icmp_reg_imm);
15098 %}
15099 
15100 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15101 %{
15102   match(Set cr (OverflowSubL op1 op2));
15103 
15104   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15105   ins_cost(INSN_COST);
15106   ins_encode %{
15107     __ cmp($op1$$Register, $op2$$Register);
15108   %}
15109 
15110   ins_pipe(icmp_reg_reg);
15111 %}
15112 
15113 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15114 %{
15115   match(Set cr (OverflowSubL op1 op2));
15116 
15117   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15118   ins_cost(INSN_COST);
15119   ins_encode %{
15120     __ subs(zr, $op1$$Register, $op2$$constant);
15121   %}
15122 
15123   ins_pipe(icmp_reg_imm);
15124 %}
15125 
15126 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15127 %{
15128   match(Set cr (OverflowSubI zero op1));
15129 
15130   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15131   ins_cost(INSN_COST);
15132   ins_encode %{
15133     __ cmpw(zr, $op1$$Register);
15134   %}
15135 
15136   ins_pipe(icmp_reg_imm);
15137 %}
15138 
15139 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15140 %{
15141   match(Set cr (OverflowSubL zero op1));
15142 
15143   format %{ "cmp   zr, $op1\t# overflow check long" %}
15144   ins_cost(INSN_COST);
15145   ins_encode %{
15146     __ cmp(zr, $op1$$Register);
15147   %}
15148 
15149   ins_pipe(icmp_reg_imm);
15150 %}
15151 
15152 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15153 %{
15154   match(Set cr (OverflowMulI op1 op2));
15155 
15156   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15157             "cmp   rscratch1, rscratch1, sxtw\n\t"
15158             "movw  rscratch1, #0x80000000\n\t"
15159             "cselw rscratch1, rscratch1, zr, NE\n\t"
15160             "cmpw  rscratch1, #1" %}
15161   ins_cost(5 * INSN_COST);
15162   ins_encode %{
15163     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15164     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15165     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15166     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15167     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15168   %}
15169 
15170   ins_pipe(pipe_slow);
15171 %}
15172 
15173 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15174 %{
15175   match(If cmp (OverflowMulI op1 op2));
15176   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15177             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15178   effect(USE labl, KILL cr);
15179 
15180   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15181             "cmp   rscratch1, rscratch1, sxtw\n\t"
15182             "b$cmp   $labl" %}
15183   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15184   ins_encode %{
15185     Label* L = $labl$$label;
15186     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15188     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15189     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15190   %}
15191 
15192   ins_pipe(pipe_serial);
15193 %}
15194 
15195 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15196 %{
15197   match(Set cr (OverflowMulL op1 op2));
15198 
15199   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15200             "smulh rscratch2, $op1, $op2\n\t"
15201             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15202             "movw  rscratch1, #0x80000000\n\t"
15203             "cselw rscratch1, rscratch1, zr, NE\n\t"
15204             "cmpw  rscratch1, #1" %}
15205   ins_cost(6 * INSN_COST);
15206   ins_encode %{
15207     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15208     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15209     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15210     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15211     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15212     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15213   %}
15214 
15215   ins_pipe(pipe_slow);
15216 %}
15217 
15218 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15219 %{
15220   match(If cmp (OverflowMulL op1 op2));
15221   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15222             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15223   effect(USE labl, KILL cr);
15224 
15225   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15226             "smulh rscratch2, $op1, $op2\n\t"
15227             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15228             "b$cmp $labl" %}
15229   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15230   ins_encode %{
15231     Label* L = $labl$$label;
15232     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15233     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15234     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15235     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15236     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15237   %}
15238 
15239   ins_pipe(pipe_serial);
15240 %}
15241 
15242 // ============================================================================
15243 // Compare Instructions
15244 
15245 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15246 %{
15247   match(Set cr (CmpI op1 op2));
15248 
15249   effect(DEF cr, USE op1, USE op2);
15250 
15251   ins_cost(INSN_COST);
15252   format %{ "cmpw  $op1, $op2" %}
15253 
15254   ins_encode(aarch64_enc_cmpw(op1, op2));
15255 
15256   ins_pipe(icmp_reg_reg);
15257 %}
15258 
15259 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15260 %{
15261   match(Set cr (CmpI op1 zero));
15262 
15263   effect(DEF cr, USE op1);
15264 
15265   ins_cost(INSN_COST);
15266   format %{ "cmpw $op1, 0" %}
15267 
15268   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15269 
15270   ins_pipe(icmp_reg_imm);
15271 %}
15272 
15273 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15274 %{
15275   match(Set cr (CmpI op1 op2));
15276 
15277   effect(DEF cr, USE op1);
15278 
15279   ins_cost(INSN_COST);
15280   format %{ "cmpw  $op1, $op2" %}
15281 
15282   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15283 
15284   ins_pipe(icmp_reg_imm);
15285 %}
15286 
15287 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15288 %{
15289   match(Set cr (CmpI op1 op2));
15290 
15291   effect(DEF cr, USE op1);
15292 
15293   ins_cost(INSN_COST * 2);
15294   format %{ "cmpw  $op1, $op2" %}
15295 
15296   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15297 
15298   ins_pipe(icmp_reg_imm);
15299 %}
15300 
15301 // Unsigned compare Instructions; really, same as signed compare
15302 // except it should only be used to feed an If or a CMovI which takes a
15303 // cmpOpU.
15304 
15305 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15306 %{
15307   match(Set cr (CmpU op1 op2));
15308 
15309   effect(DEF cr, USE op1, USE op2);
15310 
15311   ins_cost(INSN_COST);
15312   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15313 
15314   ins_encode(aarch64_enc_cmpw(op1, op2));
15315 
15316   ins_pipe(icmp_reg_reg);
15317 %}
15318 
15319 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15320 %{
15321   match(Set cr (CmpU op1 zero));
15322 
15323   effect(DEF cr, USE op1);
15324 
15325   ins_cost(INSN_COST);
15326   format %{ "cmpw $op1, #0\t# unsigned" %}
15327 
15328   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15329 
15330   ins_pipe(icmp_reg_imm);
15331 %}
15332 
15333 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15334 %{
15335   match(Set cr (CmpU op1 op2));
15336 
15337   effect(DEF cr, USE op1);
15338 
15339   ins_cost(INSN_COST);
15340   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15341 
15342   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15343 
15344   ins_pipe(icmp_reg_imm);
15345 %}
15346 
15347 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15348 %{
15349   match(Set cr (CmpU op1 op2));
15350 
15351   effect(DEF cr, USE op1);
15352 
15353   ins_cost(INSN_COST * 2);
15354   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15355 
15356   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15357 
15358   ins_pipe(icmp_reg_imm);
15359 %}
15360 
15361 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15362 %{
15363   match(Set cr (CmpL op1 op2));
15364 
15365   effect(DEF cr, USE op1, USE op2);
15366 
15367   ins_cost(INSN_COST);
15368   format %{ "cmp  $op1, $op2" %}
15369 
15370   ins_encode(aarch64_enc_cmp(op1, op2));
15371 
15372   ins_pipe(icmp_reg_reg);
15373 %}
15374 
15375 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15376 %{
15377   match(Set cr (CmpL op1 zero));
15378 
15379   effect(DEF cr, USE op1);
15380 
15381   ins_cost(INSN_COST);
15382   format %{ "tst  $op1" %}
15383 
15384   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15385 
15386   ins_pipe(icmp_reg_imm);
15387 %}
15388 
15389 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15390 %{
15391   match(Set cr (CmpL op1 op2));
15392 
15393   effect(DEF cr, USE op1);
15394 
15395   ins_cost(INSN_COST);
15396   format %{ "cmp  $op1, $op2" %}
15397 
15398   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15399 
15400   ins_pipe(icmp_reg_imm);
15401 %}
15402 
15403 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15404 %{
15405   match(Set cr (CmpL op1 op2));
15406 
15407   effect(DEF cr, USE op1);
15408 
15409   ins_cost(INSN_COST * 2);
15410   format %{ "cmp  $op1, $op2" %}
15411 
15412   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15413 
15414   ins_pipe(icmp_reg_imm);
15415 %}
15416 
15417 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15418 %{
15419   match(Set cr (CmpUL op1 op2));
15420 
15421   effect(DEF cr, USE op1, USE op2);
15422 
15423   ins_cost(INSN_COST);
15424   format %{ "cmp  $op1, $op2" %}
15425 
15426   ins_encode(aarch64_enc_cmp(op1, op2));
15427 
15428   ins_pipe(icmp_reg_reg);
15429 %}
15430 
15431 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15432 %{
15433   match(Set cr (CmpUL op1 zero));
15434 
15435   effect(DEF cr, USE op1);
15436 
15437   ins_cost(INSN_COST);
15438   format %{ "tst  $op1" %}
15439 
15440   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15441 
15442   ins_pipe(icmp_reg_imm);
15443 %}
15444 
15445 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15446 %{
15447   match(Set cr (CmpUL op1 op2));
15448 
15449   effect(DEF cr, USE op1);
15450 
15451   ins_cost(INSN_COST);
15452   format %{ "cmp  $op1, $op2" %}
15453 
15454   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15455 
15456   ins_pipe(icmp_reg_imm);
15457 %}
15458 
15459 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15460 %{
15461   match(Set cr (CmpUL op1 op2));
15462 
15463   effect(DEF cr, USE op1);
15464 
15465   ins_cost(INSN_COST * 2);
15466   format %{ "cmp  $op1, $op2" %}
15467 
15468   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15469 
15470   ins_pipe(icmp_reg_imm);
15471 %}
15472 
15473 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15474 %{
15475   match(Set cr (CmpP op1 op2));
15476 
15477   effect(DEF cr, USE op1, USE op2);
15478 
15479   ins_cost(INSN_COST);
15480   format %{ "cmp  $op1, $op2\t // ptr" %}
15481 
15482   ins_encode(aarch64_enc_cmpp(op1, op2));
15483 
15484   ins_pipe(icmp_reg_reg);
15485 %}
15486 
15487 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15488 %{
15489   match(Set cr (CmpN op1 op2));
15490 
15491   effect(DEF cr, USE op1, USE op2);
15492 
15493   ins_cost(INSN_COST);
15494   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15495 
15496   ins_encode(aarch64_enc_cmpn(op1, op2));
15497 
15498   ins_pipe(icmp_reg_reg);
15499 %}
15500 
15501 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15502 %{
15503   match(Set cr (CmpP op1 zero));
15504 
15505   effect(DEF cr, USE op1, USE zero);
15506 
15507   ins_cost(INSN_COST);
15508   format %{ "cmp  $op1, 0\t // ptr" %}
15509 
15510   ins_encode(aarch64_enc_testp(op1));
15511 
15512   ins_pipe(icmp_reg_imm);
15513 %}
15514 
15515 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15516 %{
15517   match(Set cr (CmpN op1 zero));
15518 
15519   effect(DEF cr, USE op1, USE zero);
15520 
15521   ins_cost(INSN_COST);
15522   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15523 
15524   ins_encode(aarch64_enc_testn(op1));
15525 
15526   ins_pipe(icmp_reg_imm);
15527 %}
15528 
15529 // FP comparisons
15530 //
15531 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15532 // using normal cmpOp. See declaration of rFlagsReg for details.
15533 
15534 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15535 %{
15536   match(Set cr (CmpF src1 src2));
15537 
15538   ins_cost(3 * INSN_COST);
15539   format %{ "fcmps $src1, $src2" %}
15540 
15541   ins_encode %{
15542     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15543   %}
15544 
15545   ins_pipe(pipe_class_compare);
15546 %}
15547 
15548 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15549 %{
15550   match(Set cr (CmpF src1 src2));
15551 
15552   ins_cost(3 * INSN_COST);
15553   format %{ "fcmps $src1, 0.0" %}
15554 
15555   ins_encode %{
15556     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15557   %}
15558 
15559   ins_pipe(pipe_class_compare);
15560 %}
15561 // FROM HERE
15562 
15563 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15564 %{
15565   match(Set cr (CmpD src1 src2));
15566 
15567   ins_cost(3 * INSN_COST);
15568   format %{ "fcmpd $src1, $src2" %}
15569 
15570   ins_encode %{
15571     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15572   %}
15573 
15574   ins_pipe(pipe_class_compare);
15575 %}
15576 
15577 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15578 %{
15579   match(Set cr (CmpD src1 src2));
15580 
15581   ins_cost(3 * INSN_COST);
15582   format %{ "fcmpd $src1, 0.0" %}
15583 
15584   ins_encode %{
15585     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15586   %}
15587 
15588   ins_pipe(pipe_class_compare);
15589 %}
15590 
15591 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15592 %{
15593   match(Set dst (CmpF3 src1 src2));
15594   effect(KILL cr);
15595 
15596   ins_cost(5 * INSN_COST);
15597   format %{ "fcmps $src1, $src2\n\t"
15598             "csinvw($dst, zr, zr, eq\n\t"
15599             "csnegw($dst, $dst, $dst, lt)"
15600   %}
15601 
15602   ins_encode %{
15603     Label done;
15604     FloatRegister s1 = as_FloatRegister($src1$$reg);
15605     FloatRegister s2 = as_FloatRegister($src2$$reg);
15606     Register d = as_Register($dst$$reg);
15607     __ fcmps(s1, s2);
15608     // installs 0 if EQ else -1
15609     __ csinvw(d, zr, zr, Assembler::EQ);
15610     // keeps -1 if less or unordered else installs 1
15611     __ csnegw(d, d, d, Assembler::LT);
15612     __ bind(done);
15613   %}
15614 
15615   ins_pipe(pipe_class_default);
15616 
15617 %}
15618 
15619 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15620 %{
15621   match(Set dst (CmpD3 src1 src2));
15622   effect(KILL cr);
15623 
15624   ins_cost(5 * INSN_COST);
15625   format %{ "fcmpd $src1, $src2\n\t"
15626             "csinvw($dst, zr, zr, eq\n\t"
15627             "csnegw($dst, $dst, $dst, lt)"
15628   %}
15629 
15630   ins_encode %{
15631     Label done;
15632     FloatRegister s1 = as_FloatRegister($src1$$reg);
15633     FloatRegister s2 = as_FloatRegister($src2$$reg);
15634     Register d = as_Register($dst$$reg);
15635     __ fcmpd(s1, s2);
15636     // installs 0 if EQ else -1
15637     __ csinvw(d, zr, zr, Assembler::EQ);
15638     // keeps -1 if less or unordered else installs 1
15639     __ csnegw(d, d, d, Assembler::LT);
15640     __ bind(done);
15641   %}
15642   ins_pipe(pipe_class_default);
15643 
15644 %}
15645 
15646 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15647 %{
15648   match(Set dst (CmpF3 src1 zero));
15649   effect(KILL cr);
15650 
15651   ins_cost(5 * INSN_COST);
15652   format %{ "fcmps $src1, 0.0\n\t"
15653             "csinvw($dst, zr, zr, eq\n\t"
15654             "csnegw($dst, $dst, $dst, lt)"
15655   %}
15656 
15657   ins_encode %{
15658     Label done;
15659     FloatRegister s1 = as_FloatRegister($src1$$reg);
15660     Register d = as_Register($dst$$reg);
15661     __ fcmps(s1, 0.0);
15662     // installs 0 if EQ else -1
15663     __ csinvw(d, zr, zr, Assembler::EQ);
15664     // keeps -1 if less or unordered else installs 1
15665     __ csnegw(d, d, d, Assembler::LT);
15666     __ bind(done);
15667   %}
15668 
15669   ins_pipe(pipe_class_default);
15670 
15671 %}
15672 
15673 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15674 %{
15675   match(Set dst (CmpD3 src1 zero));
15676   effect(KILL cr);
15677 
15678   ins_cost(5 * INSN_COST);
15679   format %{ "fcmpd $src1, 0.0\n\t"
15680             "csinvw($dst, zr, zr, eq\n\t"
15681             "csnegw($dst, $dst, $dst, lt)"
15682   %}
15683 
15684   ins_encode %{
15685     Label done;
15686     FloatRegister s1 = as_FloatRegister($src1$$reg);
15687     Register d = as_Register($dst$$reg);
15688     __ fcmpd(s1, 0.0);
15689     // installs 0 if EQ else -1
15690     __ csinvw(d, zr, zr, Assembler::EQ);
15691     // keeps -1 if less or unordered else installs 1
15692     __ csnegw(d, d, d, Assembler::LT);
15693     __ bind(done);
15694   %}
15695   ins_pipe(pipe_class_default);
15696 
15697 %}
15698 
15699 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15700 %{
15701   match(Set dst (CmpLTMask p q));
15702   effect(KILL cr);
15703 
15704   ins_cost(3 * INSN_COST);
15705 
15706   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15707             "csetw $dst, lt\n\t"
15708             "subw $dst, zr, $dst"
15709   %}
15710 
15711   ins_encode %{
15712     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15713     __ csetw(as_Register($dst$$reg), Assembler::LT);
15714     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15715   %}
15716 
15717   ins_pipe(ialu_reg_reg);
15718 %}
15719 
15720 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15721 %{
15722   match(Set dst (CmpLTMask src zero));
15723   effect(KILL cr);
15724 
15725   ins_cost(INSN_COST);
15726 
15727   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15728 
15729   ins_encode %{
15730     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15731   %}
15732 
15733   ins_pipe(ialu_reg_shift);
15734 %}
15735 
15736 // ============================================================================
15737 // Max and Min
15738 
15739 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15740 
15741 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15742 %{
15743   effect(DEF cr, USE src);
15744   ins_cost(INSN_COST);
15745   format %{ "cmpw $src, 0" %}
15746 
15747   ins_encode %{
15748     __ cmpw($src$$Register, 0);
15749   %}
15750   ins_pipe(icmp_reg_imm);
15751 %}
15752 
15753 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15754 %{
15755   match(Set dst (MinI src1 src2));
15756   ins_cost(INSN_COST * 3);
15757 
15758   expand %{
15759     rFlagsReg cr;
15760     compI_reg_reg(cr, src1, src2);
15761     cmovI_reg_reg_lt(dst, src1, src2, cr);
15762   %}
15763 %}
15764 
15765 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15766 %{
15767   match(Set dst (MaxI src1 src2));
15768   ins_cost(INSN_COST * 3);
15769 
15770   expand %{
15771     rFlagsReg cr;
15772     compI_reg_reg(cr, src1, src2);
15773     cmovI_reg_reg_gt(dst, src1, src2, cr);
15774   %}
15775 %}
15776 
15777 
15778 // ============================================================================
15779 // Branch Instructions
15780 
15781 // Direct Branch.
15782 instruct branch(label lbl)
15783 %{
15784   match(Goto);
15785 
15786   effect(USE lbl);
15787 
15788   ins_cost(BRANCH_COST);
15789   format %{ "b  $lbl" %}
15790 
15791   ins_encode(aarch64_enc_b(lbl));
15792 
15793   ins_pipe(pipe_branch);
15794 %}
15795 
15796 // Conditional Near Branch
15797 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15798 %{
15799   // Same match rule as `branchConFar'.
15800   match(If cmp cr);
15801 
15802   effect(USE lbl);
15803 
15804   ins_cost(BRANCH_COST);
15805   // If set to 1 this indicates that the current instruction is a
15806   // short variant of a long branch. This avoids using this
15807   // instruction in first-pass matching. It will then only be used in
15808   // the `Shorten_branches' pass.
15809   // ins_short_branch(1);
15810   format %{ "b$cmp  $lbl" %}
15811 
15812   ins_encode(aarch64_enc_br_con(cmp, lbl));
15813 
15814   ins_pipe(pipe_branch_cond);
15815 %}
15816 
15817 // Conditional Near Branch Unsigned
15818 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15819 %{
15820   // Same match rule as `branchConFar'.
15821   match(If cmp cr);
15822 
15823   effect(USE lbl);
15824 
15825   ins_cost(BRANCH_COST);
15826   // If set to 1 this indicates that the current instruction is a
15827   // short variant of a long branch. This avoids using this
15828   // instruction in first-pass matching. It will then only be used in
15829   // the `Shorten_branches' pass.
15830   // ins_short_branch(1);
15831   format %{ "b$cmp  $lbl\t# unsigned" %}
15832 
15833   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15834 
15835   ins_pipe(pipe_branch_cond);
15836 %}
15837 
15838 // Make use of CBZ and CBNZ.  These instructions, as well as being
15839 // shorter than (cmp; branch), have the additional benefit of not
15840 // killing the flags.
15841 
15842 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15843   match(If cmp (CmpI op1 op2));
15844   effect(USE labl);
15845 
15846   ins_cost(BRANCH_COST);
15847   format %{ "cbw$cmp   $op1, $labl" %}
15848   ins_encode %{
15849     Label* L = $labl$$label;
15850     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15851     if (cond == Assembler::EQ)
15852       __ cbzw($op1$$Register, *L);
15853     else
15854       __ cbnzw($op1$$Register, *L);
15855   %}
15856   ins_pipe(pipe_cmp_branch);
15857 %}
15858 
15859 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15860   match(If cmp (CmpL op1 op2));
15861   effect(USE labl);
15862 
15863   ins_cost(BRANCH_COST);
15864   format %{ "cb$cmp   $op1, $labl" %}
15865   ins_encode %{
15866     Label* L = $labl$$label;
15867     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15868     if (cond == Assembler::EQ)
15869       __ cbz($op1$$Register, *L);
15870     else
15871       __ cbnz($op1$$Register, *L);
15872   %}
15873   ins_pipe(pipe_cmp_branch);
15874 %}
15875 
15876 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15877   match(If cmp (CmpP op1 op2));
15878   effect(USE labl);
15879 
15880   ins_cost(BRANCH_COST);
15881   format %{ "cb$cmp   $op1, $labl" %}
15882   ins_encode %{
15883     Label* L = $labl$$label;
15884     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15885     if (cond == Assembler::EQ)
15886       __ cbz($op1$$Register, *L);
15887     else
15888       __ cbnz($op1$$Register, *L);
15889   %}
15890   ins_pipe(pipe_cmp_branch);
15891 %}
15892 
15893 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15894   match(If cmp (CmpN op1 op2));
15895   effect(USE labl);
15896 
15897   ins_cost(BRANCH_COST);
15898   format %{ "cbw$cmp   $op1, $labl" %}
15899   ins_encode %{
15900     Label* L = $labl$$label;
15901     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15902     if (cond == Assembler::EQ)
15903       __ cbzw($op1$$Register, *L);
15904     else
15905       __ cbnzw($op1$$Register, *L);
15906   %}
15907   ins_pipe(pipe_cmp_branch);
15908 %}
15909 
15910 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15911   match(If cmp (CmpP (DecodeN oop) zero));
15912   effect(USE labl);
15913 
15914   ins_cost(BRANCH_COST);
15915   format %{ "cb$cmp   $oop, $labl" %}
15916   ins_encode %{
15917     Label* L = $labl$$label;
15918     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15919     if (cond == Assembler::EQ)
15920       __ cbzw($oop$$Register, *L);
15921     else
15922       __ cbnzw($oop$$Register, *L);
15923   %}
15924   ins_pipe(pipe_cmp_branch);
15925 %}
15926 
15927 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15928   match(If cmp (CmpU op1 op2));
15929   effect(USE labl);
15930 
15931   ins_cost(BRANCH_COST);
15932   format %{ "cbw$cmp   $op1, $labl" %}
15933   ins_encode %{
15934     Label* L = $labl$$label;
15935     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15936     if (cond == Assembler::EQ || cond == Assembler::LS) {
15937       __ cbzw($op1$$Register, *L);
15938     } else {
15939       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15940       __ cbnzw($op1$$Register, *L);
15941     }
15942   %}
15943   ins_pipe(pipe_cmp_branch);
15944 %}
15945 
15946 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15947   match(If cmp (CmpUL op1 op2));
15948   effect(USE labl);
15949 
15950   ins_cost(BRANCH_COST);
15951   format %{ "cb$cmp   $op1, $labl" %}
15952   ins_encode %{
15953     Label* L = $labl$$label;
15954     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15955     if (cond == Assembler::EQ || cond == Assembler::LS) {
15956       __ cbz($op1$$Register, *L);
15957     } else {
15958       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15959       __ cbnz($op1$$Register, *L);
15960     }
15961   %}
15962   ins_pipe(pipe_cmp_branch);
15963 %}
15964 
15965 // Test bit and Branch
15966 
15967 // Patterns for short (< 32KiB) variants
15968 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15969   match(If cmp (CmpL op1 op2));
15970   effect(USE labl);
15971 
15972   ins_cost(BRANCH_COST);
15973   format %{ "cb$cmp   $op1, $labl # long" %}
15974   ins_encode %{
15975     Label* L = $labl$$label;
15976     Assembler::Condition cond =
15977       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15978     __ tbr(cond, $op1$$Register, 63, *L);
15979   %}
15980   ins_pipe(pipe_cmp_branch);
15981   ins_short_branch(1);
15982 %}
15983 
15984 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15985   match(If cmp (CmpI op1 op2));
15986   effect(USE labl);
15987 
15988   ins_cost(BRANCH_COST);
15989   format %{ "cb$cmp   $op1, $labl # int" %}
15990   ins_encode %{
15991     Label* L = $labl$$label;
15992     Assembler::Condition cond =
15993       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15994     __ tbr(cond, $op1$$Register, 31, *L);
15995   %}
15996   ins_pipe(pipe_cmp_branch);
15997   ins_short_branch(1);
15998 %}
15999 
16000 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16001   match(If cmp (CmpL (AndL op1 op2) op3));
16002   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16003   effect(USE labl);
16004 
16005   ins_cost(BRANCH_COST);
16006   format %{ "tb$cmp   $op1, $op2, $labl" %}
16007   ins_encode %{
16008     Label* L = $labl$$label;
16009     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16010     int bit = exact_log2_long($op2$$constant);
16011     __ tbr(cond, $op1$$Register, bit, *L);
16012   %}
16013   ins_pipe(pipe_cmp_branch);
16014   ins_short_branch(1);
16015 %}
16016 
16017 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16018   match(If cmp (CmpI (AndI op1 op2) op3));
16019   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16020   effect(USE labl);
16021 
16022   ins_cost(BRANCH_COST);
16023   format %{ "tb$cmp   $op1, $op2, $labl" %}
16024   ins_encode %{
16025     Label* L = $labl$$label;
16026     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16027     int bit = exact_log2((juint)$op2$$constant);
16028     __ tbr(cond, $op1$$Register, bit, *L);
16029   %}
16030   ins_pipe(pipe_cmp_branch);
16031   ins_short_branch(1);
16032 %}
16033 
16034 // And far variants
16035 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16036   match(If cmp (CmpL op1 op2));
16037   effect(USE labl);
16038 
16039   ins_cost(BRANCH_COST);
16040   format %{ "cb$cmp   $op1, $labl # long" %}
16041   ins_encode %{
16042     Label* L = $labl$$label;
16043     Assembler::Condition cond =
16044       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16045     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16046   %}
16047   ins_pipe(pipe_cmp_branch);
16048 %}
16049 
16050 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16051   match(If cmp (CmpI op1 op2));
16052   effect(USE labl);
16053 
16054   ins_cost(BRANCH_COST);
16055   format %{ "cb$cmp   $op1, $labl # int" %}
16056   ins_encode %{
16057     Label* L = $labl$$label;
16058     Assembler::Condition cond =
16059       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16060     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16061   %}
16062   ins_pipe(pipe_cmp_branch);
16063 %}
16064 
16065 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16066   match(If cmp (CmpL (AndL op1 op2) op3));
16067   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16068   effect(USE labl);
16069 
16070   ins_cost(BRANCH_COST);
16071   format %{ "tb$cmp   $op1, $op2, $labl" %}
16072   ins_encode %{
16073     Label* L = $labl$$label;
16074     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16075     int bit = exact_log2_long($op2$$constant);
16076     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16077   %}
16078   ins_pipe(pipe_cmp_branch);
16079 %}
16080 
16081 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16082   match(If cmp (CmpI (AndI op1 op2) op3));
16083   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16084   effect(USE labl);
16085 
16086   ins_cost(BRANCH_COST);
16087   format %{ "tb$cmp   $op1, $op2, $labl" %}
16088   ins_encode %{
16089     Label* L = $labl$$label;
16090     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16091     int bit = exact_log2((juint)$op2$$constant);
16092     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16093   %}
16094   ins_pipe(pipe_cmp_branch);
16095 %}
16096 
16097 // Test bits
16098 
16099 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16100   match(Set cr (CmpL (AndL op1 op2) op3));
16101   predicate(Assembler::operand_valid_for_logical_immediate
16102             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16103 
16104   ins_cost(INSN_COST);
16105   format %{ "tst $op1, $op2 # long" %}
16106   ins_encode %{
16107     __ tst($op1$$Register, $op2$$constant);
16108   %}
16109   ins_pipe(ialu_reg_reg);
16110 %}
16111 
16112 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16113   match(Set cr (CmpI (AndI op1 op2) op3));
16114   predicate(Assembler::operand_valid_for_logical_immediate
16115             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16116 
16117   ins_cost(INSN_COST);
16118   format %{ "tst $op1, $op2 # int" %}
16119   ins_encode %{
16120     __ tstw($op1$$Register, $op2$$constant);
16121   %}
16122   ins_pipe(ialu_reg_reg);
16123 %}
16124 
16125 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16126   match(Set cr (CmpL (AndL op1 op2) op3));
16127 
16128   ins_cost(INSN_COST);
16129   format %{ "tst $op1, $op2 # long" %}
16130   ins_encode %{
16131     __ tst($op1$$Register, $op2$$Register);
16132   %}
16133   ins_pipe(ialu_reg_reg);
16134 %}
16135 
16136 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16137   match(Set cr (CmpI (AndI op1 op2) op3));
16138 
16139   ins_cost(INSN_COST);
16140   format %{ "tstw $op1, $op2 # int" %}
16141   ins_encode %{
16142     __ tstw($op1$$Register, $op2$$Register);
16143   %}
16144   ins_pipe(ialu_reg_reg);
16145 %}
16146 
16147 
16148 // Conditional Far Branch
16149 // Conditional Far Branch Unsigned
16150 // TODO: fixme
16151 
16152 // counted loop end branch near
16153 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16154 %{
16155   match(CountedLoopEnd cmp cr);
16156 
16157   effect(USE lbl);
16158 
16159   ins_cost(BRANCH_COST);
16160   // short variant.
16161   // ins_short_branch(1);
16162   format %{ "b$cmp $lbl \t// counted loop end" %}
16163 
16164   ins_encode(aarch64_enc_br_con(cmp, lbl));
16165 
16166   ins_pipe(pipe_branch);
16167 %}
16168 
16169 // counted loop end branch far
16170 // TODO: fixme
16171 
16172 // ============================================================================
16173 // inlined locking and unlocking
16174 
16175 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16176 %{
16177   predicate(LockingMode != LM_LIGHTWEIGHT);
16178   match(Set cr (FastLock object box));
16179   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16180 
16181   ins_cost(5 * INSN_COST);
16182   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16183 
16184   ins_encode %{
16185     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16186   %}
16187 
16188   ins_pipe(pipe_serial);
16189 %}
16190 
16191 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16192 %{
16193   predicate(LockingMode != LM_LIGHTWEIGHT);
16194   match(Set cr (FastUnlock object box));
16195   effect(TEMP tmp, TEMP tmp2);
16196 
16197   ins_cost(5 * INSN_COST);
16198   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16199 
16200   ins_encode %{
16201     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16202   %}
16203 
16204   ins_pipe(pipe_serial);
16205 %}
16206 
16207 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16208 %{
16209   predicate(LockingMode == LM_LIGHTWEIGHT);
16210   match(Set cr (FastLock object box));
16211   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16212 
16213   ins_cost(5 * INSN_COST);
16214   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16215 
16216   ins_encode %{
16217     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16218   %}
16219 
16220   ins_pipe(pipe_serial);
16221 %}
16222 
16223 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16224 %{
16225   predicate(LockingMode == LM_LIGHTWEIGHT);
16226   match(Set cr (FastUnlock object box));
16227   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16228 
16229   ins_cost(5 * INSN_COST);
16230   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16231 
16232   ins_encode %{
16233     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16234   %}
16235 
16236   ins_pipe(pipe_serial);
16237 %}
16238 
16239 // ============================================================================
16240 // Safepoint Instructions
16241 
16242 // TODO
16243 // provide a near and far version of this code
16244 
16245 instruct safePoint(rFlagsReg cr, iRegP poll)
16246 %{
16247   match(SafePoint poll);
16248   effect(KILL cr);
16249 
16250   format %{
16251     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16252   %}
16253   ins_encode %{
16254     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16255   %}
16256   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16257 %}
16258 
16259 
16260 // ============================================================================
16261 // Procedure Call/Return Instructions
16262 
16263 // Call Java Static Instruction
16264 
16265 instruct CallStaticJavaDirect(method meth)
16266 %{
16267   match(CallStaticJava);
16268 
16269   effect(USE meth);
16270 
16271   ins_cost(CALL_COST);
16272 
16273   format %{ "call,static $meth \t// ==> " %}
16274 
16275   ins_encode(aarch64_enc_java_static_call(meth),
16276              aarch64_enc_call_epilog);
16277 
16278   ins_pipe(pipe_class_call);
16279 %}
16280 
16281 // TO HERE
16282 
16283 // Call Java Dynamic Instruction
16284 instruct CallDynamicJavaDirect(method meth)
16285 %{
16286   match(CallDynamicJava);
16287 
16288   effect(USE meth);
16289 
16290   ins_cost(CALL_COST);
16291 
16292   format %{ "CALL,dynamic $meth \t// ==> " %}
16293 
16294   ins_encode(aarch64_enc_java_dynamic_call(meth),
16295              aarch64_enc_call_epilog);
16296 
16297   ins_pipe(pipe_class_call);
16298 %}
16299 
16300 // Call Runtime Instruction
16301 
16302 instruct CallRuntimeDirect(method meth)
16303 %{
16304   match(CallRuntime);
16305 
16306   effect(USE meth);
16307 
16308   ins_cost(CALL_COST);
16309 
16310   format %{ "CALL, runtime $meth" %}
16311 
16312   ins_encode( aarch64_enc_java_to_runtime(meth) );
16313 
16314   ins_pipe(pipe_class_call);
16315 %}
16316 
16317 // Call Runtime Instruction
16318 
16319 instruct CallLeafDirect(method meth)
16320 %{
16321   match(CallLeaf);
16322 
16323   effect(USE meth);
16324 
16325   ins_cost(CALL_COST);
16326 
16327   format %{ "CALL, runtime leaf $meth" %}
16328 
16329   ins_encode( aarch64_enc_java_to_runtime(meth) );
16330 
16331   ins_pipe(pipe_class_call);
16332 %}
16333 
16334 // Call Runtime Instruction without safepoint and with vector arguments
16335 instruct CallLeafDirectVector(method meth)
16336 %{
16337   match(CallLeafVector);
16338 
16339   effect(USE meth);
16340 
16341   ins_cost(CALL_COST);
16342 
16343   format %{ "CALL, runtime leaf vector $meth" %}
16344 
16345   ins_encode(aarch64_enc_java_to_runtime(meth));
16346 
16347   ins_pipe(pipe_class_call);
16348 %}
16349 
16350 // Call Runtime Instruction
16351 
16352 instruct CallLeafNoFPDirect(method meth)
16353 %{
16354   match(CallLeafNoFP);
16355 
16356   effect(USE meth);
16357 
16358   ins_cost(CALL_COST);
16359 
16360   format %{ "CALL, runtime leaf nofp $meth" %}
16361 
16362   ins_encode( aarch64_enc_java_to_runtime(meth) );
16363 
16364   ins_pipe(pipe_class_call);
16365 %}
16366 
16367 // Tail Call; Jump from runtime stub to Java code.
16368 // Also known as an 'interprocedural jump'.
16369 // Target of jump will eventually return to caller.
16370 // TailJump below removes the return address.
16371 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16372 // emitted just above the TailCall which has reset rfp to the caller state.
16373 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16374 %{
16375   match(TailCall jump_target method_ptr);
16376 
16377   ins_cost(CALL_COST);
16378 
16379   format %{ "br $jump_target\t# $method_ptr holds method" %}
16380 
16381   ins_encode(aarch64_enc_tail_call(jump_target));
16382 
16383   ins_pipe(pipe_class_call);
16384 %}
16385 
16386 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16387 %{
16388   match(TailJump jump_target ex_oop);
16389 
16390   ins_cost(CALL_COST);
16391 
16392   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16393 
16394   ins_encode(aarch64_enc_tail_jmp(jump_target));
16395 
16396   ins_pipe(pipe_class_call);
16397 %}
16398 
16399 // Forward exception.
16400 instruct ForwardExceptionjmp()
16401 %{
16402   match(ForwardException);
16403   ins_cost(CALL_COST);
16404 
16405   format %{ "b forward_exception_stub" %}
16406   ins_encode %{
16407     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16408   %}
16409   ins_pipe(pipe_class_call);
16410 %}
16411 
16412 // Create exception oop: created by stack-crawling runtime code.
16413 // Created exception is now available to this handler, and is setup
16414 // just prior to jumping to this handler. No code emitted.
16415 // TODO check
16416 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16417 instruct CreateException(iRegP_R0 ex_oop)
16418 %{
16419   match(Set ex_oop (CreateEx));
16420 
16421   format %{ " -- \t// exception oop; no code emitted" %}
16422 
16423   size(0);
16424 
16425   ins_encode( /*empty*/ );
16426 
16427   ins_pipe(pipe_class_empty);
16428 %}
16429 
16430 // Rethrow exception: The exception oop will come in the first
16431 // argument position. Then JUMP (not call) to the rethrow stub code.
16432 instruct RethrowException() %{
16433   match(Rethrow);
16434   ins_cost(CALL_COST);
16435 
16436   format %{ "b rethrow_stub" %}
16437 
16438   ins_encode( aarch64_enc_rethrow() );
16439 
16440   ins_pipe(pipe_class_call);
16441 %}
16442 
16443 
16444 // Return Instruction
16445 // epilog node loads ret address into lr as part of frame pop
16446 instruct Ret()
16447 %{
16448   match(Return);
16449 
16450   format %{ "ret\t// return register" %}
16451 
16452   ins_encode( aarch64_enc_ret() );
16453 
16454   ins_pipe(pipe_branch);
16455 %}
16456 
16457 // Die now.
16458 instruct ShouldNotReachHere() %{
16459   match(Halt);
16460 
16461   ins_cost(CALL_COST);
16462   format %{ "ShouldNotReachHere" %}
16463 
16464   ins_encode %{
16465     if (is_reachable()) {
16466       const char* str = __ code_string(_halt_reason);
16467       __ stop(str);
16468     }
16469   %}
16470 
16471   ins_pipe(pipe_class_default);
16472 %}
16473 
16474 // ============================================================================
16475 // Partial Subtype Check
16476 //
16477 // superklass array for an instance of the superklass.  Set a hidden
16478 // internal cache on a hit (cache is checked with exposed code in
16479 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16480 // encoding ALSO sets flags.
16481 
16482 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16483 %{
16484   match(Set result (PartialSubtypeCheck sub super));
16485   predicate(!UseSecondarySupersTable);
16486   effect(KILL cr, KILL temp);
16487 
16488   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16489   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16490 
16491   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16492 
16493   opcode(0x1); // Force zero of result reg on hit
16494 
16495   ins_pipe(pipe_class_memory);
16496 %}
16497 
16498 // Two versions of partialSubtypeCheck, both used when we need to
16499 // search for a super class in the secondary supers array. The first
16500 // is used when we don't know _a priori_ the class being searched
16501 // for. The second, far more common, is used when we do know: this is
16502 // used for instanceof, checkcast, and any case where C2 can determine
16503 // it by constant propagation.
16504 
16505 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16506                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16507                                      rFlagsReg cr)
16508 %{
16509   match(Set result (PartialSubtypeCheck sub super));
16510   predicate(UseSecondarySupersTable);
16511   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16512 
16513   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16514   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16515 
16516   ins_encode %{
16517     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16518                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16519                                          $vtemp$$FloatRegister,
16520                                          $result$$Register, /*L_success*/nullptr);
16521   %}
16522 
16523   ins_pipe(pipe_class_memory);
16524 %}
16525 
16526 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16527                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16528                                        rFlagsReg cr)
16529 %{
16530   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16531   predicate(UseSecondarySupersTable);
16532   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16533 
16534   ins_cost(5 * INSN_COST);  // smaller than the next version
16535   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16536 
16537   ins_encode %{
16538     bool success = false;
16539     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16540     if (InlineSecondarySupersTest) {
16541       success =
16542         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16543                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16544                                                $vtemp$$FloatRegister,
16545                                                $result$$Register,
16546                                                super_klass_slot);
16547     } else {
16548       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16549       success = (call != nullptr);
16550     }
16551     if (!success) {
16552       ciEnv::current()->record_failure("CodeCache is full");
16553       return;
16554     }
16555   %}
16556 
16557   ins_pipe(pipe_class_memory);
16558 %}
16559 
16560 // Intrisics for String.compareTo()
16561 
16562 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16563                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16564 %{
16565   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16566   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16567   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16568 
16569   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16570   ins_encode %{
16571     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16572     __ string_compare($str1$$Register, $str2$$Register,
16573                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16574                       $tmp1$$Register, $tmp2$$Register,
16575                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16576   %}
16577   ins_pipe(pipe_class_memory);
16578 %}
16579 
16580 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16581                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16582 %{
16583   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16584   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16585   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16586 
16587   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16588   ins_encode %{
16589     __ string_compare($str1$$Register, $str2$$Register,
16590                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16591                       $tmp1$$Register, $tmp2$$Register,
16592                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16593   %}
16594   ins_pipe(pipe_class_memory);
16595 %}
16596 
16597 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16598                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16599                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16600 %{
16601   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16602   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16603   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16604          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16605 
16606   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16607   ins_encode %{
16608     __ string_compare($str1$$Register, $str2$$Register,
16609                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16610                       $tmp1$$Register, $tmp2$$Register,
16611                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16612                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16613   %}
16614   ins_pipe(pipe_class_memory);
16615 %}
16616 
16617 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16618                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16619                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16620 %{
16621   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16622   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16623   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16624          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16625 
16626   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16627   ins_encode %{
16628     __ string_compare($str1$$Register, $str2$$Register,
16629                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16630                       $tmp1$$Register, $tmp2$$Register,
16631                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16632                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16633   %}
16634   ins_pipe(pipe_class_memory);
16635 %}
16636 
16637 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16638 // these string_compare variants as NEON register type for convenience so that the prototype of
16639 // string_compare can be shared with all variants.
16640 
16641 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16642                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16643                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16644                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16645 %{
16646   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16647   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16648   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16649          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16650 
16651   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16652   ins_encode %{
16653     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16654     __ string_compare($str1$$Register, $str2$$Register,
16655                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16656                       $tmp1$$Register, $tmp2$$Register,
16657                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16658                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16659                       StrIntrinsicNode::LL);
16660   %}
16661   ins_pipe(pipe_class_memory);
16662 %}
16663 
16664 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16665                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16666                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16667                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16668 %{
16669   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16670   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16671   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16672          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16673 
16674   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16675   ins_encode %{
16676     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16677     __ string_compare($str1$$Register, $str2$$Register,
16678                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16679                       $tmp1$$Register, $tmp2$$Register,
16680                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16681                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16682                       StrIntrinsicNode::LU);
16683   %}
16684   ins_pipe(pipe_class_memory);
16685 %}
16686 
16687 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16688                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16689                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16690                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16691 %{
16692   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16693   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16694   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16695          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16696 
16697   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16698   ins_encode %{
16699     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16700     __ string_compare($str1$$Register, $str2$$Register,
16701                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16702                       $tmp1$$Register, $tmp2$$Register,
16703                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16704                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16705                       StrIntrinsicNode::UL);
16706   %}
16707   ins_pipe(pipe_class_memory);
16708 %}
16709 
16710 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16711                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16712                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16713                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16714 %{
16715   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16716   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16717   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16718          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16719 
16720   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16721   ins_encode %{
16722     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16723     __ string_compare($str1$$Register, $str2$$Register,
16724                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16725                       $tmp1$$Register, $tmp2$$Register,
16726                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16727                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16728                       StrIntrinsicNode::UU);
16729   %}
16730   ins_pipe(pipe_class_memory);
16731 %}
16732 
16733 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16734                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16735                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16736                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16737 %{
16738   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16739   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16740   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16741          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16742          TEMP vtmp0, TEMP vtmp1, KILL cr);
16743   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16744             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16745 
16746   ins_encode %{
16747     __ string_indexof($str1$$Register, $str2$$Register,
16748                       $cnt1$$Register, $cnt2$$Register,
16749                       $tmp1$$Register, $tmp2$$Register,
16750                       $tmp3$$Register, $tmp4$$Register,
16751                       $tmp5$$Register, $tmp6$$Register,
16752                       -1, $result$$Register, StrIntrinsicNode::UU);
16753   %}
16754   ins_pipe(pipe_class_memory);
16755 %}
16756 
16757 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16758                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16759                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16760                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16761 %{
16762   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16763   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16764   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16765          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16766          TEMP vtmp0, TEMP vtmp1, KILL cr);
16767   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16768             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16769 
16770   ins_encode %{
16771     __ string_indexof($str1$$Register, $str2$$Register,
16772                       $cnt1$$Register, $cnt2$$Register,
16773                       $tmp1$$Register, $tmp2$$Register,
16774                       $tmp3$$Register, $tmp4$$Register,
16775                       $tmp5$$Register, $tmp6$$Register,
16776                       -1, $result$$Register, StrIntrinsicNode::LL);
16777   %}
16778   ins_pipe(pipe_class_memory);
16779 %}
16780 
16781 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16782                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16783                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16784                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16785 %{
16786   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16787   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16788   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16789          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16790          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16791   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16792             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16793 
16794   ins_encode %{
16795     __ string_indexof($str1$$Register, $str2$$Register,
16796                       $cnt1$$Register, $cnt2$$Register,
16797                       $tmp1$$Register, $tmp2$$Register,
16798                       $tmp3$$Register, $tmp4$$Register,
16799                       $tmp5$$Register, $tmp6$$Register,
16800                       -1, $result$$Register, StrIntrinsicNode::UL);
16801   %}
16802   ins_pipe(pipe_class_memory);
16803 %}
16804 
16805 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16806                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16807                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16808 %{
16809   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16810   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16811   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16812          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16813   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16814             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16815 
16816   ins_encode %{
16817     int icnt2 = (int)$int_cnt2$$constant;
16818     __ string_indexof($str1$$Register, $str2$$Register,
16819                       $cnt1$$Register, zr,
16820                       $tmp1$$Register, $tmp2$$Register,
16821                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16822                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16823   %}
16824   ins_pipe(pipe_class_memory);
16825 %}
16826 
16827 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16828                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16829                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16830 %{
16831   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16832   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16833   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16834          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16835   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16836             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16837 
16838   ins_encode %{
16839     int icnt2 = (int)$int_cnt2$$constant;
16840     __ string_indexof($str1$$Register, $str2$$Register,
16841                       $cnt1$$Register, zr,
16842                       $tmp1$$Register, $tmp2$$Register,
16843                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16844                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16845   %}
16846   ins_pipe(pipe_class_memory);
16847 %}
16848 
16849 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16850                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16851                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16852 %{
16853   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16854   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16855   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16856          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16857   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16858             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16859 
16860   ins_encode %{
16861     int icnt2 = (int)$int_cnt2$$constant;
16862     __ string_indexof($str1$$Register, $str2$$Register,
16863                       $cnt1$$Register, zr,
16864                       $tmp1$$Register, $tmp2$$Register,
16865                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16866                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16867   %}
16868   ins_pipe(pipe_class_memory);
16869 %}
16870 
16871 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16872                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16873                              iRegINoSp tmp3, rFlagsReg cr)
16874 %{
16875   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16876   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16877   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16878          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16879 
16880   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16881 
16882   ins_encode %{
16883     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16884                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16885                            $tmp3$$Register);
16886   %}
16887   ins_pipe(pipe_class_memory);
16888 %}
16889 
16890 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16891                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16892                               iRegINoSp tmp3, rFlagsReg cr)
16893 %{
16894   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16895   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16896   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16897          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16898 
16899   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16900 
16901   ins_encode %{
16902     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16903                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16904                             $tmp3$$Register);
16905   %}
16906   ins_pipe(pipe_class_memory);
16907 %}
16908 
16909 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16910                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16911                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16912   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16913   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16914   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16915   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16916   ins_encode %{
16917     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16918                                $result$$Register, $ztmp1$$FloatRegister,
16919                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16920                                $ptmp$$PRegister, true /* isL */);
16921   %}
16922   ins_pipe(pipe_class_memory);
16923 %}
16924 
16925 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16926                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16927                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16928   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16929   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16930   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16931   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16932   ins_encode %{
16933     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16934                                $result$$Register, $ztmp1$$FloatRegister,
16935                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16936                                $ptmp$$PRegister, false /* isL */);
16937   %}
16938   ins_pipe(pipe_class_memory);
16939 %}
16940 
16941 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16942                         iRegI_R0 result, rFlagsReg cr)
16943 %{
16944   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16945   match(Set result (StrEquals (Binary str1 str2) cnt));
16946   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16947 
16948   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16949   ins_encode %{
16950     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16951     __ string_equals($str1$$Register, $str2$$Register,
16952                      $result$$Register, $cnt$$Register);
16953   %}
16954   ins_pipe(pipe_class_memory);
16955 %}
16956 
16957 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16958                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16959                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16960                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16961                        iRegP_R10 tmp, rFlagsReg cr)
16962 %{
16963   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16964   match(Set result (AryEq ary1 ary2));
16965   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16966          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16967          TEMP vtmp6, TEMP vtmp7, KILL cr);
16968 
16969   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16970   ins_encode %{
16971     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16972                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16973                                    $result$$Register, $tmp$$Register, 1);
16974     if (tpc == nullptr) {
16975       ciEnv::current()->record_failure("CodeCache is full");
16976       return;
16977     }
16978   %}
16979   ins_pipe(pipe_class_memory);
16980 %}
16981 
16982 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16983                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16984                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16985                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16986                        iRegP_R10 tmp, rFlagsReg cr)
16987 %{
16988   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16989   match(Set result (AryEq ary1 ary2));
16990   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16991          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16992          TEMP vtmp6, TEMP vtmp7, KILL cr);
16993 
16994   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16995   ins_encode %{
16996     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16997                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16998                                    $result$$Register, $tmp$$Register, 2);
16999     if (tpc == nullptr) {
17000       ciEnv::current()->record_failure("CodeCache is full");
17001       return;
17002     }
17003   %}
17004   ins_pipe(pipe_class_memory);
17005 %}
17006 
17007 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17008                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17009                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17010                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17011 %{
17012   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17013   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17014          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17015 
17016   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
17017   ins_encode %{
17018     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17019                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17020                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17021                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17022                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17023                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17024                                      (BasicType)$basic_type$$constant);
17025     if (tpc == nullptr) {
17026       ciEnv::current()->record_failure("CodeCache is full");
17027       return;
17028     }
17029   %}
17030   ins_pipe(pipe_class_memory);
17031 %}
17032 
17033 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17034 %{
17035   match(Set result (CountPositives ary1 len));
17036   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17037   format %{ "count positives byte[] $ary1,$len -> $result" %}
17038   ins_encode %{
17039     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17040     if (tpc == nullptr) {
17041       ciEnv::current()->record_failure("CodeCache is full");
17042       return;
17043     }
17044   %}
17045   ins_pipe( pipe_slow );
17046 %}
17047 
17048 // fast char[] to byte[] compression
17049 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17050                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17051                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17052                          iRegI_R0 result, rFlagsReg cr)
17053 %{
17054   match(Set result (StrCompressedCopy src (Binary dst len)));
17055   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17056          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17057 
17058   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17059   ins_encode %{
17060     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17061                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17062                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17063                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17064   %}
17065   ins_pipe(pipe_slow);
17066 %}
17067 
17068 // fast byte[] to char[] inflation
17069 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17070                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17071                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17072 %{
17073   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17074   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17075          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17076          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17077 
17078   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17079   ins_encode %{
17080     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17081                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17082                                         $vtmp2$$FloatRegister, $tmp$$Register);
17083     if (tpc == nullptr) {
17084       ciEnv::current()->record_failure("CodeCache is full");
17085       return;
17086     }
17087   %}
17088   ins_pipe(pipe_class_memory);
17089 %}
17090 
17091 // encode char[] to byte[] in ISO_8859_1
17092 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17093                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17094                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17095                           iRegI_R0 result, rFlagsReg cr)
17096 %{
17097   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17098   match(Set result (EncodeISOArray src (Binary dst len)));
17099   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17100          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17101 
17102   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17103   ins_encode %{
17104     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17105                         $result$$Register, false,
17106                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17107                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17108                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17109   %}
17110   ins_pipe(pipe_class_memory);
17111 %}
17112 
17113 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17114                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17115                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17116                             iRegI_R0 result, rFlagsReg cr)
17117 %{
17118   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17119   match(Set result (EncodeISOArray src (Binary dst len)));
17120   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17121          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17122 
17123   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17124   ins_encode %{
17125     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17126                         $result$$Register, true,
17127                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17128                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17129                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17130   %}
17131   ins_pipe(pipe_class_memory);
17132 %}
17133 
17134 //----------------------------- CompressBits/ExpandBits ------------------------
17135 
17136 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17137                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17138   match(Set dst (CompressBits src mask));
17139   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17140   format %{ "mov    $tsrc, $src\n\t"
17141             "mov    $tmask, $mask\n\t"
17142             "bext   $tdst, $tsrc, $tmask\n\t"
17143             "mov    $dst, $tdst"
17144           %}
17145   ins_encode %{
17146     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17147     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17148     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17149     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17150   %}
17151   ins_pipe(pipe_slow);
17152 %}
17153 
17154 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17155                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17156   match(Set dst (CompressBits (LoadI mem) mask));
17157   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17158   format %{ "ldrs   $tsrc, $mem\n\t"
17159             "ldrs   $tmask, $mask\n\t"
17160             "bext   $tdst, $tsrc, $tmask\n\t"
17161             "mov    $dst, $tdst"
17162           %}
17163   ins_encode %{
17164     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17165               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17166     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17167     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17168     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17169   %}
17170   ins_pipe(pipe_slow);
17171 %}
17172 
17173 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17174                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17175   match(Set dst (CompressBits src mask));
17176   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17177   format %{ "mov    $tsrc, $src\n\t"
17178             "mov    $tmask, $mask\n\t"
17179             "bext   $tdst, $tsrc, $tmask\n\t"
17180             "mov    $dst, $tdst"
17181           %}
17182   ins_encode %{
17183     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17184     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17185     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17186     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17187   %}
17188   ins_pipe(pipe_slow);
17189 %}
17190 
17191 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17192                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17193   match(Set dst (CompressBits (LoadL mem) mask));
17194   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17195   format %{ "ldrd   $tsrc, $mem\n\t"
17196             "ldrd   $tmask, $mask\n\t"
17197             "bext   $tdst, $tsrc, $tmask\n\t"
17198             "mov    $dst, $tdst"
17199           %}
17200   ins_encode %{
17201     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17202               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17203     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17204     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17205     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17206   %}
17207   ins_pipe(pipe_slow);
17208 %}
17209 
17210 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17211                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17212   match(Set dst (ExpandBits src mask));
17213   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17214   format %{ "mov    $tsrc, $src\n\t"
17215             "mov    $tmask, $mask\n\t"
17216             "bdep   $tdst, $tsrc, $tmask\n\t"
17217             "mov    $dst, $tdst"
17218           %}
17219   ins_encode %{
17220     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17221     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17222     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17223     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17224   %}
17225   ins_pipe(pipe_slow);
17226 %}
17227 
17228 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17229                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17230   match(Set dst (ExpandBits (LoadI mem) mask));
17231   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17232   format %{ "ldrs   $tsrc, $mem\n\t"
17233             "ldrs   $tmask, $mask\n\t"
17234             "bdep   $tdst, $tsrc, $tmask\n\t"
17235             "mov    $dst, $tdst"
17236           %}
17237   ins_encode %{
17238     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17239               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17240     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17241     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17242     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17243   %}
17244   ins_pipe(pipe_slow);
17245 %}
17246 
17247 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17248                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17249   match(Set dst (ExpandBits src mask));
17250   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17251   format %{ "mov    $tsrc, $src\n\t"
17252             "mov    $tmask, $mask\n\t"
17253             "bdep   $tdst, $tsrc, $tmask\n\t"
17254             "mov    $dst, $tdst"
17255           %}
17256   ins_encode %{
17257     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17258     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17259     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17260     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17261   %}
17262   ins_pipe(pipe_slow);
17263 %}
17264 
17265 
17266 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17267                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17268   match(Set dst (ExpandBits (LoadL mem) mask));
17269   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17270   format %{ "ldrd   $tsrc, $mem\n\t"
17271             "ldrd   $tmask, $mask\n\t"
17272             "bdep   $tdst, $tsrc, $tmask\n\t"
17273             "mov    $dst, $tdst"
17274           %}
17275   ins_encode %{
17276     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17277               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17278     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17279     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17280     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17281   %}
17282   ins_pipe(pipe_slow);
17283 %}
17284 
17285 //----------------------------- Reinterpret ----------------------------------
17286 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17287 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17288   match(Set dst (ReinterpretHF2S src));
17289   format %{ "reinterpretHF2S $dst, $src" %}
17290   ins_encode %{
17291     __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17292   %}
17293   ins_pipe(pipe_slow);
17294 %}
17295 
17296 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17297 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17298   match(Set dst (ReinterpretS2HF src));
17299   format %{ "reinterpretS2HF $dst, $src" %}
17300   ins_encode %{
17301     __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17302   %}
17303   ins_pipe(pipe_slow);
17304 %}
17305 
17306 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17307 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17308 // fcvt $tmp1_fpr, $src_fpr    // Convert float to half-precision float
17309 // mov  $tmp2_gpr, $tmp1_fpr   // Move half-precision float in FPR to a GPR
17310 // mov  $dst_fpr,  $tmp2_gpr   // Move the result from a GPR to an FPR
17311 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17312 // can be omitted in this pattern, resulting in -
17313 // fcvt $dst, $src  // Convert float to half-precision float
17314 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17315 %{
17316   match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17317   format %{ "convF2HFAndS2HF $dst, $src" %}
17318   ins_encode %{
17319     __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17320   %}
17321   ins_pipe(pipe_slow);
17322 %}
17323 
17324 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17325 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17326 // mov  $tmp1_gpr, $src_fpr  // Move the half-precision float from an FPR to a GPR
17327 // mov  $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17328 // fcvt $dst_fpr,  $tmp2_fpr // Convert the half-precision float to 32-bit float
17329 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17330 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17331 // resulting in -
17332 // fcvt $dst, $src  // Convert half-precision float to a 32-bit float
17333 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17334 %{
17335   match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17336   format %{ "convHF2SAndHF2F $dst, $src" %}
17337   ins_encode %{
17338     __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17339   %}
17340   ins_pipe(pipe_slow);
17341 %}
17342 
17343 // ============================================================================
17344 // This name is KNOWN by the ADLC and cannot be changed.
17345 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17346 // for this guy.
17347 instruct tlsLoadP(thread_RegP dst)
17348 %{
17349   match(Set dst (ThreadLocal));
17350 
17351   ins_cost(0);
17352 
17353   format %{ " -- \t// $dst=Thread::current(), empty" %}
17354 
17355   size(0);
17356 
17357   ins_encode( /*empty*/ );
17358 
17359   ins_pipe(pipe_class_empty);
17360 %}
17361 
17362 //----------PEEPHOLE RULES-----------------------------------------------------
17363 // These must follow all instruction definitions as they use the names
17364 // defined in the instructions definitions.
17365 //
17366 // peepmatch ( root_instr_name [preceding_instruction]* );
17367 //
17368 // peepconstraint %{
17369 // (instruction_number.operand_name relational_op instruction_number.operand_name
17370 //  [, ...] );
17371 // // instruction numbers are zero-based using left to right order in peepmatch
17372 //
17373 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17374 // // provide an instruction_number.operand_name for each operand that appears
17375 // // in the replacement instruction's match rule
17376 //
17377 // ---------VM FLAGS---------------------------------------------------------
17378 //
17379 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17380 //
17381 // Each peephole rule is given an identifying number starting with zero and
17382 // increasing by one in the order seen by the parser.  An individual peephole
17383 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17384 // on the command-line.
17385 //
17386 // ---------CURRENT LIMITATIONS----------------------------------------------
17387 //
17388 // Only match adjacent instructions in same basic block
17389 // Only equality constraints
17390 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17391 // Only one replacement instruction
17392 //
17393 // ---------EXAMPLE----------------------------------------------------------
17394 //
17395 // // pertinent parts of existing instructions in architecture description
17396 // instruct movI(iRegINoSp dst, iRegI src)
17397 // %{
17398 //   match(Set dst (CopyI src));
17399 // %}
17400 //
17401 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17402 // %{
17403 //   match(Set dst (AddI dst src));
17404 //   effect(KILL cr);
17405 // %}
17406 //
17407 // // Change (inc mov) to lea
17408 // peephole %{
17409 //   // increment preceded by register-register move
17410 //   peepmatch ( incI_iReg movI );
17411 //   // require that the destination register of the increment
17412 //   // match the destination register of the move
17413 //   peepconstraint ( 0.dst == 1.dst );
17414 //   // construct a replacement instruction that sets
17415 //   // the destination to ( move's source register + one )
17416 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17417 // %}
17418 //
17419 
17420 // Implementation no longer uses movX instructions since
17421 // machine-independent system no longer uses CopyX nodes.
17422 //
17423 // peephole
17424 // %{
17425 //   peepmatch (incI_iReg movI);
17426 //   peepconstraint (0.dst == 1.dst);
17427 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17428 // %}
17429 
17430 // peephole
17431 // %{
17432 //   peepmatch (decI_iReg movI);
17433 //   peepconstraint (0.dst == 1.dst);
17434 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17435 // %}
17436 
17437 // peephole
17438 // %{
17439 //   peepmatch (addI_iReg_imm movI);
17440 //   peepconstraint (0.dst == 1.dst);
17441 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17442 // %}
17443 
17444 // peephole
17445 // %{
17446 //   peepmatch (incL_iReg movL);
17447 //   peepconstraint (0.dst == 1.dst);
17448 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17449 // %}
17450 
17451 // peephole
17452 // %{
17453 //   peepmatch (decL_iReg movL);
17454 //   peepconstraint (0.dst == 1.dst);
17455 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17456 // %}
17457 
17458 // peephole
17459 // %{
17460 //   peepmatch (addL_iReg_imm movL);
17461 //   peepconstraint (0.dst == 1.dst);
17462 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17463 // %}
17464 
17465 // peephole
17466 // %{
17467 //   peepmatch (addP_iReg_imm movP);
17468 //   peepconstraint (0.dst == 1.dst);
17469 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17470 // %}
17471 
17472 // // Change load of spilled value to only a spill
17473 // instruct storeI(memory mem, iRegI src)
17474 // %{
17475 //   match(Set mem (StoreI mem src));
17476 // %}
17477 //
17478 // instruct loadI(iRegINoSp dst, memory mem)
17479 // %{
17480 //   match(Set dst (LoadI mem));
17481 // %}
17482 //
17483 
17484 //----------SMARTSPILL RULES---------------------------------------------------
17485 // These must follow all instruction definitions as they use the names
17486 // defined in the instructions definitions.
17487 
17488 // Local Variables:
17489 // mode: c++
17490 // End: