1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all float registers
  698 reg_class float_reg(
  699     V0,
  700     V1,
  701     V2,
  702     V3,
  703     V4,
  704     V5,
  705     V6,
  706     V7,
  707     V8,
  708     V9,
  709     V10,
  710     V11,
  711     V12,
  712     V13,
  713     V14,
  714     V15,
  715     V16,
  716     V17,
  717     V18,
  718     V19,
  719     V20,
  720     V21,
  721     V22,
  722     V23,
  723     V24,
  724     V25,
  725     V26,
  726     V27,
  727     V28,
  728     V29,
  729     V30,
  730     V31
  731 );
  732 
  733 // Double precision float registers have virtual `high halves' that
  734 // are needed by the allocator.
  735 // Class for all double registers
  736 reg_class double_reg(
  737     V0, V0_H,
  738     V1, V1_H,
  739     V2, V2_H,
  740     V3, V3_H,
  741     V4, V4_H,
  742     V5, V5_H,
  743     V6, V6_H,
  744     V7, V7_H,
  745     V8, V8_H,
  746     V9, V9_H,
  747     V10, V10_H,
  748     V11, V11_H,
  749     V12, V12_H,
  750     V13, V13_H,
  751     V14, V14_H,
  752     V15, V15_H,
  753     V16, V16_H,
  754     V17, V17_H,
  755     V18, V18_H,
  756     V19, V19_H,
  757     V20, V20_H,
  758     V21, V21_H,
  759     V22, V22_H,
  760     V23, V23_H,
  761     V24, V24_H,
  762     V25, V25_H,
  763     V26, V26_H,
  764     V27, V27_H,
  765     V28, V28_H,
  766     V29, V29_H,
  767     V30, V30_H,
  768     V31, V31_H
  769 );
  770 
  771 // Class for all SVE vector registers.
  772 reg_class vectora_reg (
  773     V0, V0_H, V0_J, V0_K,
  774     V1, V1_H, V1_J, V1_K,
  775     V2, V2_H, V2_J, V2_K,
  776     V3, V3_H, V3_J, V3_K,
  777     V4, V4_H, V4_J, V4_K,
  778     V5, V5_H, V5_J, V5_K,
  779     V6, V6_H, V6_J, V6_K,
  780     V7, V7_H, V7_J, V7_K,
  781     V8, V8_H, V8_J, V8_K,
  782     V9, V9_H, V9_J, V9_K,
  783     V10, V10_H, V10_J, V10_K,
  784     V11, V11_H, V11_J, V11_K,
  785     V12, V12_H, V12_J, V12_K,
  786     V13, V13_H, V13_J, V13_K,
  787     V14, V14_H, V14_J, V14_K,
  788     V15, V15_H, V15_J, V15_K,
  789     V16, V16_H, V16_J, V16_K,
  790     V17, V17_H, V17_J, V17_K,
  791     V18, V18_H, V18_J, V18_K,
  792     V19, V19_H, V19_J, V19_K,
  793     V20, V20_H, V20_J, V20_K,
  794     V21, V21_H, V21_J, V21_K,
  795     V22, V22_H, V22_J, V22_K,
  796     V23, V23_H, V23_J, V23_K,
  797     V24, V24_H, V24_J, V24_K,
  798     V25, V25_H, V25_J, V25_K,
  799     V26, V26_H, V26_J, V26_K,
  800     V27, V27_H, V27_J, V27_K,
  801     V28, V28_H, V28_J, V28_K,
  802     V29, V29_H, V29_J, V29_K,
  803     V30, V30_H, V30_J, V30_K,
  804     V31, V31_H, V31_J, V31_K,
  805 );
  806 
  807 // Class for all 64bit vector registers
  808 reg_class vectord_reg(
  809     V0, V0_H,
  810     V1, V1_H,
  811     V2, V2_H,
  812     V3, V3_H,
  813     V4, V4_H,
  814     V5, V5_H,
  815     V6, V6_H,
  816     V7, V7_H,
  817     V8, V8_H,
  818     V9, V9_H,
  819     V10, V10_H,
  820     V11, V11_H,
  821     V12, V12_H,
  822     V13, V13_H,
  823     V14, V14_H,
  824     V15, V15_H,
  825     V16, V16_H,
  826     V17, V17_H,
  827     V18, V18_H,
  828     V19, V19_H,
  829     V20, V20_H,
  830     V21, V21_H,
  831     V22, V22_H,
  832     V23, V23_H,
  833     V24, V24_H,
  834     V25, V25_H,
  835     V26, V26_H,
  836     V27, V27_H,
  837     V28, V28_H,
  838     V29, V29_H,
  839     V30, V30_H,
  840     V31, V31_H
  841 );
  842 
  843 // Class for all 128bit vector registers
  844 reg_class vectorx_reg(
  845     V0, V0_H, V0_J, V0_K,
  846     V1, V1_H, V1_J, V1_K,
  847     V2, V2_H, V2_J, V2_K,
  848     V3, V3_H, V3_J, V3_K,
  849     V4, V4_H, V4_J, V4_K,
  850     V5, V5_H, V5_J, V5_K,
  851     V6, V6_H, V6_J, V6_K,
  852     V7, V7_H, V7_J, V7_K,
  853     V8, V8_H, V8_J, V8_K,
  854     V9, V9_H, V9_J, V9_K,
  855     V10, V10_H, V10_J, V10_K,
  856     V11, V11_H, V11_J, V11_K,
  857     V12, V12_H, V12_J, V12_K,
  858     V13, V13_H, V13_J, V13_K,
  859     V14, V14_H, V14_J, V14_K,
  860     V15, V15_H, V15_J, V15_K,
  861     V16, V16_H, V16_J, V16_K,
  862     V17, V17_H, V17_J, V17_K,
  863     V18, V18_H, V18_J, V18_K,
  864     V19, V19_H, V19_J, V19_K,
  865     V20, V20_H, V20_J, V20_K,
  866     V21, V21_H, V21_J, V21_K,
  867     V22, V22_H, V22_J, V22_K,
  868     V23, V23_H, V23_J, V23_K,
  869     V24, V24_H, V24_J, V24_K,
  870     V25, V25_H, V25_J, V25_K,
  871     V26, V26_H, V26_J, V26_K,
  872     V27, V27_H, V27_J, V27_K,
  873     V28, V28_H, V28_J, V28_K,
  874     V29, V29_H, V29_J, V29_K,
  875     V30, V30_H, V30_J, V30_K,
  876     V31, V31_H, V31_J, V31_K
  877 );
  878 
  879 // Class for 128 bit register v0
  880 reg_class v0_reg(
  881     V0, V0_H
  882 );
  883 
  884 // Class for 128 bit register v1
  885 reg_class v1_reg(
  886     V1, V1_H
  887 );
  888 
  889 // Class for 128 bit register v2
  890 reg_class v2_reg(
  891     V2, V2_H
  892 );
  893 
  894 // Class for 128 bit register v3
  895 reg_class v3_reg(
  896     V3, V3_H
  897 );
  898 
  899 // Class for 128 bit register v4
  900 reg_class v4_reg(
  901     V4, V4_H
  902 );
  903 
  904 // Class for 128 bit register v5
  905 reg_class v5_reg(
  906     V5, V5_H
  907 );
  908 
  909 // Class for 128 bit register v6
  910 reg_class v6_reg(
  911     V6, V6_H
  912 );
  913 
  914 // Class for 128 bit register v7
  915 reg_class v7_reg(
  916     V7, V7_H
  917 );
  918 
  919 // Class for 128 bit register v8
  920 reg_class v8_reg(
  921     V8, V8_H
  922 );
  923 
  924 // Class for 128 bit register v9
  925 reg_class v9_reg(
  926     V9, V9_H
  927 );
  928 
  929 // Class for 128 bit register v10
  930 reg_class v10_reg(
  931     V10, V10_H
  932 );
  933 
  934 // Class for 128 bit register v11
  935 reg_class v11_reg(
  936     V11, V11_H
  937 );
  938 
  939 // Class for 128 bit register v12
  940 reg_class v12_reg(
  941     V12, V12_H
  942 );
  943 
  944 // Class for 128 bit register v13
  945 reg_class v13_reg(
  946     V13, V13_H
  947 );
  948 
  949 // Class for 128 bit register v14
  950 reg_class v14_reg(
  951     V14, V14_H
  952 );
  953 
  954 // Class for 128 bit register v15
  955 reg_class v15_reg(
  956     V15, V15_H
  957 );
  958 
  959 // Class for 128 bit register v16
  960 reg_class v16_reg(
  961     V16, V16_H
  962 );
  963 
  964 // Class for 128 bit register v17
  965 reg_class v17_reg(
  966     V17, V17_H
  967 );
  968 
  969 // Class for 128 bit register v18
  970 reg_class v18_reg(
  971     V18, V18_H
  972 );
  973 
  974 // Class for 128 bit register v19
  975 reg_class v19_reg(
  976     V19, V19_H
  977 );
  978 
  979 // Class for 128 bit register v20
  980 reg_class v20_reg(
  981     V20, V20_H
  982 );
  983 
  984 // Class for 128 bit register v21
  985 reg_class v21_reg(
  986     V21, V21_H
  987 );
  988 
  989 // Class for 128 bit register v22
  990 reg_class v22_reg(
  991     V22, V22_H
  992 );
  993 
  994 // Class for 128 bit register v23
  995 reg_class v23_reg(
  996     V23, V23_H
  997 );
  998 
  999 // Class for 128 bit register v24
 1000 reg_class v24_reg(
 1001     V24, V24_H
 1002 );
 1003 
 1004 // Class for 128 bit register v25
 1005 reg_class v25_reg(
 1006     V25, V25_H
 1007 );
 1008 
 1009 // Class for 128 bit register v26
 1010 reg_class v26_reg(
 1011     V26, V26_H
 1012 );
 1013 
 1014 // Class for 128 bit register v27
 1015 reg_class v27_reg(
 1016     V27, V27_H
 1017 );
 1018 
 1019 // Class for 128 bit register v28
 1020 reg_class v28_reg(
 1021     V28, V28_H
 1022 );
 1023 
 1024 // Class for 128 bit register v29
 1025 reg_class v29_reg(
 1026     V29, V29_H
 1027 );
 1028 
 1029 // Class for 128 bit register v30
 1030 reg_class v30_reg(
 1031     V30, V30_H
 1032 );
 1033 
 1034 // Class for 128 bit register v31
 1035 reg_class v31_reg(
 1036     V31, V31_H
 1037 );
 1038 
 1039 // Class for all SVE predicate registers.
 1040 reg_class pr_reg (
 1041     P0,
 1042     P1,
 1043     P2,
 1044     P3,
 1045     P4,
 1046     P5,
 1047     P6,
 1048     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1049     P8,
 1050     P9,
 1051     P10,
 1052     P11,
 1053     P12,
 1054     P13,
 1055     P14,
 1056     P15
 1057 );
 1058 
 1059 // Class for SVE governing predicate registers, which are used
 1060 // to determine the active elements of a predicated instruction.
 1061 reg_class gov_pr (
 1062     P0,
 1063     P1,
 1064     P2,
 1065     P3,
 1066     P4,
 1067     P5,
 1068     P6,
 1069     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1070 );
 1071 
 1072 reg_class p0_reg(P0);
 1073 reg_class p1_reg(P1);
 1074 
 1075 // Singleton class for condition codes
 1076 reg_class int_flags(RFLAGS);
 1077 
 1078 %}
 1079 
 1080 //----------DEFINITION BLOCK---------------------------------------------------
 1081 // Define name --> value mappings to inform the ADLC of an integer valued name
 1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1083 // Format:
 1084 //        int_def  <name>         ( <int_value>, <expression>);
 1085 // Generated Code in ad_<arch>.hpp
 1086 //        #define  <name>   (<expression>)
 1087 //        // value == <int_value>
 1088 // Generated code in ad_<arch>.cpp adlc_verification()
 1089 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1090 //
 1091 
 1092 // we follow the ppc-aix port in using a simple cost model which ranks
 1093 // register operations as cheap, memory ops as more expensive and
 1094 // branches as most expensive. the first two have a low as well as a
 1095 // normal cost. huge cost appears to be a way of saying don't do
 1096 // something
 1097 
 1098 definitions %{
 1099   // The default cost (of a register move instruction).
 1100   int_def INSN_COST            (    100,     100);
 1101   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1102   int_def CALL_COST            (    200,     2 * INSN_COST);
 1103   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1104 %}
 1105 
 1106 
 1107 //----------SOURCE BLOCK-------------------------------------------------------
 1108 // This is a block of C++ code which provides values, functions, and
 1109 // definitions necessary in the rest of the architecture description
 1110 
 1111 source_hpp %{
 1112 
 1113 #include "asm/macroAssembler.hpp"
 1114 #include "gc/shared/barrierSetAssembler.hpp"
 1115 #include "gc/shared/cardTable.hpp"
 1116 #include "gc/shared/cardTableBarrierSet.hpp"
 1117 #include "gc/shared/collectedHeap.hpp"
 1118 #include "opto/addnode.hpp"
 1119 #include "opto/convertnode.hpp"
 1120 #include "runtime/objectMonitor.hpp"
 1121 
 1122 extern RegMask _ANY_REG32_mask;
 1123 extern RegMask _ANY_REG_mask;
 1124 extern RegMask _PTR_REG_mask;
 1125 extern RegMask _NO_SPECIAL_REG32_mask;
 1126 extern RegMask _NO_SPECIAL_REG_mask;
 1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1128 
 1129 class CallStubImpl {
 1130 
 1131   //--------------------------------------------------------------
 1132   //---<  Used for optimization in Compile::shorten_branches  >---
 1133   //--------------------------------------------------------------
 1134 
 1135  public:
 1136   // Size of call trampoline stub.
 1137   static uint size_call_trampoline() {
 1138     return 0; // no call trampolines on this platform
 1139   }
 1140 
 1141   // number of relocations needed by a call trampoline stub
 1142   static uint reloc_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 };
 1146 
 1147 class HandlerImpl {
 1148 
 1149  public:
 1150 
 1151   static int emit_exception_handler(CodeBuffer &cbuf);
 1152   static int emit_deopt_handler(CodeBuffer& cbuf);
 1153 
 1154   static uint size_exception_handler() {
 1155     return MacroAssembler::far_codestub_branch_size();
 1156   }
 1157 
 1158   static uint size_deopt_handler() {
 1159     // count one adr and one far branch instruction
 1160     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 };
 1163 
 1164 class Node::PD {
 1165 public:
 1166   enum NodeFlags {
 1167     _last_flag = Node::_last_flag
 1168   };
 1169 };
 1170 
 1171   bool is_CAS(int opcode, bool maybe_volatile);
 1172 
 1173   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1174 
 1175   bool unnecessary_acquire(const Node *barrier);
 1176   bool needs_acquiring_load(const Node *load);
 1177 
 1178   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1179 
 1180   bool unnecessary_release(const Node *barrier);
 1181   bool unnecessary_volatile(const Node *barrier);
 1182   bool needs_releasing_store(const Node *store);
 1183 
 1184   // predicate controlling translation of CompareAndSwapX
 1185   bool needs_acquiring_load_exclusive(const Node *load);
 1186 
 1187   // predicate controlling addressing modes
 1188   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1189 
 1190   // Convert BootTest condition to Assembler condition.
 1191   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1192   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1193 %}
 1194 
 1195 source %{
 1196 
 1197   // Derived RegMask with conditionally allocatable registers
 1198 
 1199   void PhaseOutput::pd_perform_mach_node_analysis() {
 1200   }
 1201 
 1202   int MachNode::pd_alignment_required() const {
 1203     return 1;
 1204   }
 1205 
 1206   int MachNode::compute_padding(int current_offset) const {
 1207     return 0;
 1208   }
 1209 
 1210   RegMask _ANY_REG32_mask;
 1211   RegMask _ANY_REG_mask;
 1212   RegMask _PTR_REG_mask;
 1213   RegMask _NO_SPECIAL_REG32_mask;
 1214   RegMask _NO_SPECIAL_REG_mask;
 1215   RegMask _NO_SPECIAL_PTR_REG_mask;
 1216 
 1217   void reg_mask_init() {
 1218     // We derive below RegMask(s) from the ones which are auto-generated from
 1219     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1220     // registers conditionally reserved.
 1221 
 1222     _ANY_REG32_mask = _ALL_REG32_mask;
 1223     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1224 
 1225     _ANY_REG_mask = _ALL_REG_mask;
 1226 
 1227     _PTR_REG_mask = _ALL_REG_mask;
 1228 
 1229     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1230     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1231 
 1232     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1233     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1234 
 1235     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1236     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1237 
 1238     // r27 is not allocatable when compressed oops is on and heapbase is not
 1239     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1240     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1241       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1242       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1244     }
 1245 
 1246     // r29 is not allocatable when PreserveFramePointer is on
 1247     if (PreserveFramePointer) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1251     }
 1252   }
 1253 
 1254   // Optimizaton of volatile gets and puts
 1255   // -------------------------------------
 1256   //
 1257   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1258   // use to implement volatile reads and writes. For a volatile read
 1259   // we simply need
 1260   //
 1261   //   ldar<x>
 1262   //
 1263   // and for a volatile write we need
 1264   //
 1265   //   stlr<x>
 1266   //
 1267   // Alternatively, we can implement them by pairing a normal
 1268   // load/store with a memory barrier. For a volatile read we need
 1269   //
 1270   //   ldr<x>
 1271   //   dmb ishld
 1272   //
 1273   // for a volatile write
 1274   //
 1275   //   dmb ish
 1276   //   str<x>
 1277   //   dmb ish
 1278   //
 1279   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1280   // sequences. These are normally translated to an instruction
 1281   // sequence like the following
 1282   //
 1283   //   dmb      ish
 1284   // retry:
 1285   //   ldxr<x>   rval raddr
 1286   //   cmp       rval rold
 1287   //   b.ne done
 1288   //   stlxr<x>  rval, rnew, rold
 1289   //   cbnz      rval retry
 1290   // done:
 1291   //   cset      r0, eq
 1292   //   dmb ishld
 1293   //
 1294   // Note that the exclusive store is already using an stlxr
 1295   // instruction. That is required to ensure visibility to other
 1296   // threads of the exclusive write (assuming it succeeds) before that
 1297   // of any subsequent writes.
 1298   //
 1299   // The following instruction sequence is an improvement on the above
 1300   //
 1301   // retry:
 1302   //   ldaxr<x>  rval raddr
 1303   //   cmp       rval rold
 1304   //   b.ne done
 1305   //   stlxr<x>  rval, rnew, rold
 1306   //   cbnz      rval retry
 1307   // done:
 1308   //   cset      r0, eq
 1309   //
 1310   // We don't need the leading dmb ish since the stlxr guarantees
 1311   // visibility of prior writes in the case that the swap is
 1312   // successful. Crucially we don't have to worry about the case where
 1313   // the swap is not successful since no valid program should be
 1314   // relying on visibility of prior changes by the attempting thread
 1315   // in the case where the CAS fails.
 1316   //
 1317   // Similarly, we don't need the trailing dmb ishld if we substitute
 1318   // an ldaxr instruction since that will provide all the guarantees we
 1319   // require regarding observation of changes made by other threads
 1320   // before any change to the CAS address observed by the load.
 1321   //
 1322   // In order to generate the desired instruction sequence we need to
 1323   // be able to identify specific 'signature' ideal graph node
 1324   // sequences which i) occur as a translation of a volatile reads or
 1325   // writes or CAS operations and ii) do not occur through any other
 1326   // translation or graph transformation. We can then provide
 1327   // alternative aldc matching rules which translate these node
 1328   // sequences to the desired machine code sequences. Selection of the
 1329   // alternative rules can be implemented by predicates which identify
 1330   // the relevant node sequences.
 1331   //
 1332   // The ideal graph generator translates a volatile read to the node
 1333   // sequence
 1334   //
 1335   //   LoadX[mo_acquire]
 1336   //   MemBarAcquire
 1337   //
 1338   // As a special case when using the compressed oops optimization we
 1339   // may also see this variant
 1340   //
 1341   //   LoadN[mo_acquire]
 1342   //   DecodeN
 1343   //   MemBarAcquire
 1344   //
 1345   // A volatile write is translated to the node sequence
 1346   //
 1347   //   MemBarRelease
 1348   //   StoreX[mo_release] {CardMark}-optional
 1349   //   MemBarVolatile
 1350   //
 1351   // n.b. the above node patterns are generated with a strict
 1352   // 'signature' configuration of input and output dependencies (see
 1353   // the predicates below for exact details). The card mark may be as
 1354   // simple as a few extra nodes or, in a few GC configurations, may
 1355   // include more complex control flow between the leading and
 1356   // trailing memory barriers. However, whatever the card mark
 1357   // configuration these signatures are unique to translated volatile
 1358   // reads/stores -- they will not appear as a result of any other
 1359   // bytecode translation or inlining nor as a consequence of
 1360   // optimizing transforms.
 1361   //
 1362   // We also want to catch inlined unsafe volatile gets and puts and
 1363   // be able to implement them using either ldar<x>/stlr<x> or some
 1364   // combination of ldr<x>/stlr<x> and dmb instructions.
 1365   //
 1366   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1367   // normal volatile put node sequence containing an extra cpuorder
 1368   // membar
 1369   //
 1370   //   MemBarRelease
 1371   //   MemBarCPUOrder
 1372   //   StoreX[mo_release] {CardMark}-optional
 1373   //   MemBarCPUOrder
 1374   //   MemBarVolatile
 1375   //
 1376   // n.b. as an aside, a cpuorder membar is not itself subject to
 1377   // matching and translation by adlc rules.  However, the rule
 1378   // predicates need to detect its presence in order to correctly
 1379   // select the desired adlc rules.
 1380   //
 1381   // Inlined unsafe volatile gets manifest as a slightly different
 1382   // node sequence to a normal volatile get because of the
 1383   // introduction of some CPUOrder memory barriers to bracket the
 1384   // Load. However, but the same basic skeleton of a LoadX feeding a
 1385   // MemBarAcquire, possibly through an optional DecodeN, is still
 1386   // present
 1387   //
 1388   //   MemBarCPUOrder
 1389   //        ||       \\
 1390   //   MemBarCPUOrder LoadX[mo_acquire]
 1391   //        ||            |
 1392   //        ||       {DecodeN} optional
 1393   //        ||       /
 1394   //     MemBarAcquire
 1395   //
 1396   // In this case the acquire membar does not directly depend on the
 1397   // load. However, we can be sure that the load is generated from an
 1398   // inlined unsafe volatile get if we see it dependent on this unique
 1399   // sequence of membar nodes. Similarly, given an acquire membar we
 1400   // can know that it was added because of an inlined unsafe volatile
 1401   // get if it is fed and feeds a cpuorder membar and if its feed
 1402   // membar also feeds an acquiring load.
 1403   //
 1404   // Finally an inlined (Unsafe) CAS operation is translated to the
 1405   // following ideal graph
 1406   //
 1407   //   MemBarRelease
 1408   //   MemBarCPUOrder
 1409   //   CompareAndSwapX {CardMark}-optional
 1410   //   MemBarCPUOrder
 1411   //   MemBarAcquire
 1412   //
 1413   // So, where we can identify these volatile read and write
 1414   // signatures we can choose to plant either of the above two code
 1415   // sequences. For a volatile read we can simply plant a normal
 1416   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1417   // also choose to inhibit translation of the MemBarAcquire and
 1418   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1419   //
 1420   // When we recognise a volatile store signature we can choose to
 1421   // plant at a dmb ish as a translation for the MemBarRelease, a
 1422   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1423   // Alternatively, we can inhibit translation of the MemBarRelease
 1424   // and MemBarVolatile and instead plant a simple stlr<x>
 1425   // instruction.
 1426   //
 1427   // when we recognise a CAS signature we can choose to plant a dmb
 1428   // ish as a translation for the MemBarRelease, the conventional
 1429   // macro-instruction sequence for the CompareAndSwap node (which
 1430   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1431   // Alternatively, we can elide generation of the dmb instructions
 1432   // and plant the alternative CompareAndSwap macro-instruction
 1433   // sequence (which uses ldaxr<x>).
 1434   //
 1435   // Of course, the above only applies when we see these signature
 1436   // configurations. We still want to plant dmb instructions in any
 1437   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1438   // MemBarVolatile. For example, at the end of a constructor which
 1439   // writes final/volatile fields we will see a MemBarRelease
 1440   // instruction and this needs a 'dmb ish' lest we risk the
 1441   // constructed object being visible without making the
 1442   // final/volatile field writes visible.
 1443   //
 1444   // n.b. the translation rules below which rely on detection of the
 1445   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1446   // If we see anything other than the signature configurations we
 1447   // always just translate the loads and stores to ldr<x> and str<x>
 1448   // and translate acquire, release and volatile membars to the
 1449   // relevant dmb instructions.
 1450   //
 1451 
 1452   // is_CAS(int opcode, bool maybe_volatile)
 1453   //
 1454   // return true if opcode is one of the possible CompareAndSwapX
 1455   // values otherwise false.
 1456 
 1457   bool is_CAS(int opcode, bool maybe_volatile)
 1458   {
 1459     switch(opcode) {
 1460       // We handle these
 1461     case Op_CompareAndSwapI:
 1462     case Op_CompareAndSwapL:
 1463     case Op_CompareAndSwapP:
 1464     case Op_CompareAndSwapN:
 1465     case Op_ShenandoahCompareAndSwapP:
 1466     case Op_ShenandoahCompareAndSwapN:
 1467     case Op_CompareAndSwapB:
 1468     case Op_CompareAndSwapS:
 1469     case Op_GetAndSetI:
 1470     case Op_GetAndSetL:
 1471     case Op_GetAndSetP:
 1472     case Op_GetAndSetN:
 1473     case Op_GetAndAddI:
 1474     case Op_GetAndAddL:
 1475       return true;
 1476     case Op_CompareAndExchangeI:
 1477     case Op_CompareAndExchangeN:
 1478     case Op_CompareAndExchangeB:
 1479     case Op_CompareAndExchangeS:
 1480     case Op_CompareAndExchangeL:
 1481     case Op_CompareAndExchangeP:
 1482     case Op_WeakCompareAndSwapB:
 1483     case Op_WeakCompareAndSwapS:
 1484     case Op_WeakCompareAndSwapI:
 1485     case Op_WeakCompareAndSwapL:
 1486     case Op_WeakCompareAndSwapP:
 1487     case Op_WeakCompareAndSwapN:
 1488     case Op_ShenandoahWeakCompareAndSwapP:
 1489     case Op_ShenandoahWeakCompareAndSwapN:
 1490     case Op_ShenandoahCompareAndExchangeP:
 1491     case Op_ShenandoahCompareAndExchangeN:
 1492       return maybe_volatile;
 1493     default:
 1494       return false;
 1495     }
 1496   }
 1497 
 1498   // helper to determine the maximum number of Phi nodes we may need to
 1499   // traverse when searching from a card mark membar for the merge mem
 1500   // feeding a trailing membar or vice versa
 1501 
 1502 // predicates controlling emit of ldr<x>/ldar<x>
 1503 
 1504 bool unnecessary_acquire(const Node *barrier)
 1505 {
 1506   assert(barrier->is_MemBar(), "expecting a membar");
 1507 
 1508   MemBarNode* mb = barrier->as_MemBar();
 1509 
 1510   if (mb->trailing_load()) {
 1511     return true;
 1512   }
 1513 
 1514   if (mb->trailing_load_store()) {
 1515     Node* load_store = mb->in(MemBarNode::Precedent);
 1516     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1517     return is_CAS(load_store->Opcode(), true);
 1518   }
 1519 
 1520   return false;
 1521 }
 1522 
 1523 bool needs_acquiring_load(const Node *n)
 1524 {
 1525   assert(n->is_Load(), "expecting a load");
 1526   LoadNode *ld = n->as_Load();
 1527   return ld->is_acquire();
 1528 }
 1529 
 1530 bool unnecessary_release(const Node *n)
 1531 {
 1532   assert((n->is_MemBar() &&
 1533           n->Opcode() == Op_MemBarRelease),
 1534          "expecting a release membar");
 1535 
 1536   MemBarNode *barrier = n->as_MemBar();
 1537   if (!barrier->leading()) {
 1538     return false;
 1539   } else {
 1540     Node* trailing = barrier->trailing_membar();
 1541     MemBarNode* trailing_mb = trailing->as_MemBar();
 1542     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1543     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1544 
 1545     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1546     if (mem->is_Store()) {
 1547       assert(mem->as_Store()->is_release(), "");
 1548       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1549       return true;
 1550     } else {
 1551       assert(mem->is_LoadStore(), "");
 1552       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1553       return is_CAS(mem->Opcode(), true);
 1554     }
 1555   }
 1556   return false;
 1557 }
 1558 
 1559 bool unnecessary_volatile(const Node *n)
 1560 {
 1561   // assert n->is_MemBar();
 1562   MemBarNode *mbvol = n->as_MemBar();
 1563 
 1564   bool release = mbvol->trailing_store();
 1565   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1566 #ifdef ASSERT
 1567   if (release) {
 1568     Node* leading = mbvol->leading_membar();
 1569     assert(leading->Opcode() == Op_MemBarRelease, "");
 1570     assert(leading->as_MemBar()->leading_store(), "");
 1571     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1572   }
 1573 #endif
 1574 
 1575   return release;
 1576 }
 1577 
 1578 // predicates controlling emit of str<x>/stlr<x>
 1579 
 1580 bool needs_releasing_store(const Node *n)
 1581 {
 1582   // assert n->is_Store();
 1583   StoreNode *st = n->as_Store();
 1584   return st->trailing_membar() != NULL;
 1585 }
 1586 
 1587 // predicate controlling translation of CAS
 1588 //
 1589 // returns true if CAS needs to use an acquiring load otherwise false
 1590 
 1591 bool needs_acquiring_load_exclusive(const Node *n)
 1592 {
 1593   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1594   LoadStoreNode* ldst = n->as_LoadStore();
 1595   if (is_CAS(n->Opcode(), false)) {
 1596     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1597   } else {
 1598     return ldst->trailing_membar() != NULL;
 1599   }
 1600 
 1601   // so we can just return true here
 1602   return true;
 1603 }
 1604 
 1605 #define __ _masm.
 1606 
 1607 // advance declarations for helper functions to convert register
 1608 // indices to register objects
 1609 
 1610 // the ad file has to provide implementations of certain methods
 1611 // expected by the generic code
 1612 //
 1613 // REQUIRED FUNCTIONALITY
 1614 
 1615 //=============================================================================
 1616 
 1617 // !!!!! Special hack to get all types of calls to specify the byte offset
 1618 //       from the start of the call to the point where the return address
 1619 //       will point.
 1620 
 1621 int MachCallStaticJavaNode::ret_addr_offset()
 1622 {
 1623   // call should be a simple bl
 1624   int off = 4;
 1625   return off;
 1626 }
 1627 
 1628 int MachCallDynamicJavaNode::ret_addr_offset()
 1629 {
 1630   return 16; // movz, movk, movk, bl
 1631 }
 1632 
 1633 int MachCallRuntimeNode::ret_addr_offset() {
 1634   // for generated stubs the call will be
 1635   //   bl(addr)
 1636   // or with far branches
 1637   //   bl(trampoline_stub)
 1638   // for real runtime callouts it will be six instructions
 1639   // see aarch64_enc_java_to_runtime
 1640   //   adr(rscratch2, retaddr)
 1641   //   lea(rscratch1, RuntimeAddress(addr)
 1642   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1643   //   blr(rscratch1)
 1644   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1645   if (cb) {
 1646     return 1 * NativeInstruction::instruction_size;
 1647   } else {
 1648     return 6 * NativeInstruction::instruction_size;
 1649   }
 1650 }
 1651 
 1652 //=============================================================================
 1653 
 1654 #ifndef PRODUCT
 1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1656   st->print("BREAKPOINT");
 1657 }
 1658 #endif
 1659 
 1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1661   C2_MacroAssembler _masm(&cbuf);
 1662   __ brk(0);
 1663 }
 1664 
 1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1666   return MachNode::size(ra_);
 1667 }
 1668 
 1669 //=============================================================================
 1670 
 1671 #ifndef PRODUCT
 1672   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1673     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1674   }
 1675 #endif
 1676 
 1677   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1678     C2_MacroAssembler _masm(&cbuf);
 1679     for (int i = 0; i < _count; i++) {
 1680       __ nop();
 1681     }
 1682   }
 1683 
 1684   uint MachNopNode::size(PhaseRegAlloc*) const {
 1685     return _count * NativeInstruction::instruction_size;
 1686   }
 1687 
 1688 //=============================================================================
 1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1690 
 1691 int ConstantTable::calculate_table_base_offset() const {
 1692   return 0;  // absolute addressing, no offset
 1693 }
 1694 
 1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1697   ShouldNotReachHere();
 1698 }
 1699 
 1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1701   // Empty encoding
 1702 }
 1703 
 1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1705   return 0;
 1706 }
 1707 
 1708 #ifndef PRODUCT
 1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1710   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1711 }
 1712 #endif
 1713 
 1714 #ifndef PRODUCT
 1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1716   Compile* C = ra_->C;
 1717 
 1718   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1719 
 1720   if (C->output()->need_stack_bang(framesize))
 1721     st->print("# stack bang size=%d\n\t", framesize);
 1722 
 1723   if (VM_Version::use_rop_protection()) {
 1724     st->print("ldr zr, [lr]\n\t");
 1725     st->print("pacia  lr, rfp\n\t");
 1726   }
 1727   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1728     st->print("sub  sp, sp, #%d\n\t", framesize);
 1729     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1730     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1731   } else {
 1732     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1733     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1734     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1735     st->print("sub  sp, sp, rscratch1");
 1736   }
 1737   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1738     st->print("\n\t");
 1739     st->print("ldr  rscratch1, [guard]\n\t");
 1740     st->print("dmb ishld\n\t");
 1741     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1742     st->print("cmp  rscratch1, rscratch2\n\t");
 1743     st->print("b.eq skip");
 1744     st->print("\n\t");
 1745     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1746     st->print("b skip\n\t");
 1747     st->print("guard: int\n\t");
 1748     st->print("\n\t");
 1749     st->print("skip:\n\t");
 1750   }
 1751 }
 1752 #endif
 1753 
 1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1755   Compile* C = ra_->C;
 1756   C2_MacroAssembler _masm(&cbuf);
 1757 
 1758   // n.b. frame size includes space for return pc and rfp
 1759   const int framesize = C->output()->frame_size_in_bytes();
 1760 
 1761   // insert a nop at the start of the prolog so we can patch in a
 1762   // branch if we need to invalidate the method later
 1763   __ nop();
 1764 
 1765   if (C->clinit_barrier_on_entry()) {
 1766     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1767 
 1768     Label L_skip_barrier;
 1769 
 1770     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1771     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1772     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1773     __ bind(L_skip_barrier);
 1774   }
 1775 
 1776   if (C->max_vector_size() > 0) {
 1777     __ reinitialize_ptrue();
 1778   }
 1779 
 1780   int bangsize = C->output()->bang_size_in_bytes();
 1781   if (C->output()->need_stack_bang(bangsize))
 1782     __ generate_stack_overflow_check(bangsize);
 1783 
 1784   __ build_frame(framesize);
 1785 
 1786   if (C->stub_function() == NULL) {
 1787     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1788     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1789       // Dummy labels for just measuring the code size
 1790       Label dummy_slow_path;
 1791       Label dummy_continuation;
 1792       Label dummy_guard;
 1793       Label* slow_path = &dummy_slow_path;
 1794       Label* continuation = &dummy_continuation;
 1795       Label* guard = &dummy_guard;
 1796       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1797         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1798         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1799         Compile::current()->output()->add_stub(stub);
 1800         slow_path = &stub->entry();
 1801         continuation = &stub->continuation();
 1802         guard = &stub->guard();
 1803       }
 1804       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1805       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1806     }
 1807   }
 1808 
 1809   if (VerifyStackAtCalls) {
 1810     Unimplemented();
 1811   }
 1812 
 1813   C->output()->set_frame_complete(cbuf.insts_size());
 1814 
 1815   if (C->has_mach_constant_base_node()) {
 1816     // NOTE: We set the table base offset here because users might be
 1817     // emitted before MachConstantBaseNode.
 1818     ConstantTable& constant_table = C->output()->constant_table();
 1819     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1820   }
 1821 }
 1822 
 1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1824 {
 1825   return MachNode::size(ra_); // too many variables; just compute it
 1826                               // the hard way
 1827 }
 1828 
 1829 int MachPrologNode::reloc() const
 1830 {
 1831   return 0;
 1832 }
 1833 
 1834 //=============================================================================
 1835 
 1836 #ifndef PRODUCT
 1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1838   Compile* C = ra_->C;
 1839   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1840 
 1841   st->print("# pop frame %d\n\t",framesize);
 1842 
 1843   if (framesize == 0) {
 1844     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1845   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1846     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1847     st->print("add  sp, sp, #%d\n\t", framesize);
 1848   } else {
 1849     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1850     st->print("add  sp, sp, rscratch1\n\t");
 1851     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1852   }
 1853   if (VM_Version::use_rop_protection()) {
 1854     st->print("autia lr, rfp\n\t");
 1855     st->print("ldr zr, [lr]\n\t");
 1856   }
 1857 
 1858   if (do_polling() && C->is_method_compilation()) {
 1859     st->print("# test polling word\n\t");
 1860     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1861     st->print("cmp  sp, rscratch1\n\t");
 1862     st->print("bhi #slow_path");
 1863   }
 1864 }
 1865 #endif
 1866 
 1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1868   Compile* C = ra_->C;
 1869   C2_MacroAssembler _masm(&cbuf);
 1870   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1871 
 1872   __ remove_frame(framesize);
 1873 
 1874   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1875     __ reserved_stack_check();
 1876   }
 1877 
 1878   if (do_polling() && C->is_method_compilation()) {
 1879     Label dummy_label;
 1880     Label* code_stub = &dummy_label;
 1881     if (!C->output()->in_scratch_emit_size()) {
 1882       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1883       C->output()->add_stub(stub);
 1884       code_stub = &stub->entry();
 1885     }
 1886     __ relocate(relocInfo::poll_return_type);
 1887     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1888   }
 1889 }
 1890 
 1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1892   // Variable size. Determine dynamically.
 1893   return MachNode::size(ra_);
 1894 }
 1895 
 1896 int MachEpilogNode::reloc() const {
 1897   // Return number of relocatable values contained in this instruction.
 1898   return 1; // 1 for polling page.
 1899 }
 1900 
 1901 const Pipeline * MachEpilogNode::pipeline() const {
 1902   return MachNode::pipeline_class();
 1903 }
 1904 
 1905 //=============================================================================
 1906 
 1907 // Figure out which register class each belongs in: rc_int, rc_float or
 1908 // rc_stack.
 1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1910 
 1911 static enum RC rc_class(OptoReg::Name reg) {
 1912 
 1913   if (reg == OptoReg::Bad) {
 1914     return rc_bad;
 1915   }
 1916 
 1917   // we have 32 int registers * 2 halves
 1918   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1919 
 1920   if (reg < slots_of_int_registers) {
 1921     return rc_int;
 1922   }
 1923 
 1924   // we have 32 float register * 8 halves
 1925   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1926   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1927     return rc_float;
 1928   }
 1929 
 1930   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1931   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1932     return rc_predicate;
 1933   }
 1934 
 1935   // Between predicate regs & stack is the flags.
 1936   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1937 
 1938   return rc_stack;
 1939 }
 1940 
 1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1942   Compile* C = ra_->C;
 1943 
 1944   // Get registers to move.
 1945   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1946   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1947   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1948   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1949 
 1950   enum RC src_hi_rc = rc_class(src_hi);
 1951   enum RC src_lo_rc = rc_class(src_lo);
 1952   enum RC dst_hi_rc = rc_class(dst_hi);
 1953   enum RC dst_lo_rc = rc_class(dst_lo);
 1954 
 1955   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1956 
 1957   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1958     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1959            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1960            "expected aligned-adjacent pairs");
 1961   }
 1962 
 1963   if (src_lo == dst_lo && src_hi == dst_hi) {
 1964     return 0;            // Self copy, no move.
 1965   }
 1966 
 1967   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1968               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1969   int src_offset = ra_->reg2offset(src_lo);
 1970   int dst_offset = ra_->reg2offset(dst_lo);
 1971 
 1972   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1973     uint ireg = ideal_reg();
 1974     if (ireg == Op_VecA && cbuf) {
 1975       C2_MacroAssembler _masm(cbuf);
 1976       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1977       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1978         // stack->stack
 1979         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1980                                                 sve_vector_reg_size_in_bytes);
 1981       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1982         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1983                             sve_vector_reg_size_in_bytes);
 1984       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1985         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1986                               sve_vector_reg_size_in_bytes);
 1987       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1988         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1989                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1990                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1991       } else {
 1992         ShouldNotReachHere();
 1993       }
 1994     } else if (cbuf) {
 1995       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1996       C2_MacroAssembler _masm(cbuf);
 1997       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1998       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1999         // stack->stack
 2000         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2001         if (ireg == Op_VecD) {
 2002           __ unspill(rscratch1, true, src_offset);
 2003           __ spill(rscratch1, true, dst_offset);
 2004         } else {
 2005           __ spill_copy128(src_offset, dst_offset);
 2006         }
 2007       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2008         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2009                ireg == Op_VecD ? __ T8B : __ T16B,
 2010                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2011       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2012         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2013                  ireg == Op_VecD ? __ D : __ Q,
 2014                  ra_->reg2offset(dst_lo));
 2015       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2016         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                    ireg == Op_VecD ? __ D : __ Q,
 2018                    ra_->reg2offset(src_lo));
 2019       } else {
 2020         ShouldNotReachHere();
 2021       }
 2022     }
 2023   } else if (cbuf) {
 2024     C2_MacroAssembler _masm(cbuf);
 2025     switch (src_lo_rc) {
 2026     case rc_int:
 2027       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2028         if (is64) {
 2029             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2030                    as_Register(Matcher::_regEncode[src_lo]));
 2031         } else {
 2032             C2_MacroAssembler _masm(cbuf);
 2033             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2034                     as_Register(Matcher::_regEncode[src_lo]));
 2035         }
 2036       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2037         if (is64) {
 2038             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2039                      as_Register(Matcher::_regEncode[src_lo]));
 2040         } else {
 2041             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2042                      as_Register(Matcher::_regEncode[src_lo]));
 2043         }
 2044       } else {                    // gpr --> stack spill
 2045         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2046         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2047       }
 2048       break;
 2049     case rc_float:
 2050       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2051         if (is64) {
 2052             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2053                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2054         } else {
 2055             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2056                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2057         }
 2058       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2059         if (is64) {
 2060             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2061                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2062         } else {
 2063             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2064                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2065         }
 2066       } else {                    // fpr --> stack spill
 2067         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2068         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2069                  is64 ? __ D : __ S, dst_offset);
 2070       }
 2071       break;
 2072     case rc_stack:
 2073       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2074         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2075       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2076         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2077                    is64 ? __ D : __ S, src_offset);
 2078       } else if (dst_lo_rc == rc_predicate) {
 2079         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2080                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2081       } else {                    // stack --> stack copy
 2082         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2083         if (ideal_reg() == Op_RegVectMask) {
 2084           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2085                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2086         } else {
 2087           __ unspill(rscratch1, is64, src_offset);
 2088           __ spill(rscratch1, is64, dst_offset);
 2089         }
 2090       }
 2091       break;
 2092     case rc_predicate:
 2093       if (dst_lo_rc == rc_predicate) {
 2094         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2095       } else if (dst_lo_rc == rc_stack) {
 2096         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2097                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2098       } else {
 2099         assert(false, "bad src and dst rc_class combination.");
 2100         ShouldNotReachHere();
 2101       }
 2102       break;
 2103     default:
 2104       assert(false, "bad rc_class for spill");
 2105       ShouldNotReachHere();
 2106     }
 2107   }
 2108 
 2109   if (st) {
 2110     st->print("spill ");
 2111     if (src_lo_rc == rc_stack) {
 2112       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2113     } else {
 2114       st->print("%s -> ", Matcher::regName[src_lo]);
 2115     }
 2116     if (dst_lo_rc == rc_stack) {
 2117       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2118     } else {
 2119       st->print("%s", Matcher::regName[dst_lo]);
 2120     }
 2121     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2122       int vsize = 0;
 2123       switch (ideal_reg()) {
 2124       case Op_VecD:
 2125         vsize = 64;
 2126         break;
 2127       case Op_VecX:
 2128         vsize = 128;
 2129         break;
 2130       case Op_VecA:
 2131         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2132         break;
 2133       default:
 2134         assert(false, "bad register type for spill");
 2135         ShouldNotReachHere();
 2136       }
 2137       st->print("\t# vector spill size = %d", vsize);
 2138     } else if (ideal_reg() == Op_RegVectMask) {
 2139       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2140       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2141       st->print("\t# predicate spill size = %d", vsize);
 2142     } else {
 2143       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2144     }
 2145   }
 2146 
 2147   return 0;
 2148 
 2149 }
 2150 
 2151 #ifndef PRODUCT
 2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2153   if (!ra_)
 2154     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2155   else
 2156     implementation(NULL, ra_, false, st);
 2157 }
 2158 #endif
 2159 
 2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2161   implementation(&cbuf, ra_, false, NULL);
 2162 }
 2163 
 2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2165   return MachNode::size(ra_);
 2166 }
 2167 
 2168 //=============================================================================
 2169 
 2170 #ifndef PRODUCT
 2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2172   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2173   int reg = ra_->get_reg_first(this);
 2174   st->print("add %s, rsp, #%d]\t# box lock",
 2175             Matcher::regName[reg], offset);
 2176 }
 2177 #endif
 2178 
 2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2180   C2_MacroAssembler _masm(&cbuf);
 2181 
 2182   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2183   int reg    = ra_->get_encode(this);
 2184 
 2185   // This add will handle any 24-bit signed offset. 24 bits allows an
 2186   // 8 megabyte stack frame.
 2187   __ add(as_Register(reg), sp, offset);
 2188 }
 2189 
 2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2191   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2192   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2193 
 2194   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2195     return NativeInstruction::instruction_size;
 2196   } else {
 2197     return 2 * NativeInstruction::instruction_size;
 2198   }
 2199 }
 2200 
 2201 //=============================================================================
 2202 
 2203 #ifndef PRODUCT
 2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2205 {
 2206   st->print_cr("# MachUEPNode");
 2207   if (UseCompressedClassPointers) {
 2208     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2209     if (CompressedKlassPointers::shift() != 0) {
 2210       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2211     }
 2212   } else {
 2213    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2214   }
 2215   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2216   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2217 }
 2218 #endif
 2219 
 2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2221 {
 2222   // This is the unverified entry point.
 2223   C2_MacroAssembler _masm(&cbuf);
 2224 
 2225   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2226   Label skip;
 2227   // TODO
 2228   // can we avoid this skip and still use a reloc?
 2229   __ br(Assembler::EQ, skip);
 2230   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2231   __ bind(skip);
 2232 }
 2233 
 2234 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2235 {
 2236   return MachNode::size(ra_);
 2237 }
 2238 
 2239 // REQUIRED EMIT CODE
 2240 
 2241 //=============================================================================
 2242 
 2243 // Emit exception handler code.
 2244 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2245 {
 2246   // mov rscratch1 #exception_blob_entry_point
 2247   // br rscratch1
 2248   // Note that the code buffer's insts_mark is always relative to insts.
 2249   // That's why we must use the macroassembler to generate a handler.
 2250   C2_MacroAssembler _masm(&cbuf);
 2251   address base = __ start_a_stub(size_exception_handler());
 2252   if (base == NULL) {
 2253     ciEnv::current()->record_failure("CodeCache is full");
 2254     return 0;  // CodeBuffer::expand failed
 2255   }
 2256   int offset = __ offset();
 2257   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2258   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2259   __ end_a_stub();
 2260   return offset;
 2261 }
 2262 
 2263 // Emit deopt handler code.
 2264 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2265 {
 2266   // Note that the code buffer's insts_mark is always relative to insts.
 2267   // That's why we must use the macroassembler to generate a handler.
 2268   C2_MacroAssembler _masm(&cbuf);
 2269   address base = __ start_a_stub(size_deopt_handler());
 2270   if (base == NULL) {
 2271     ciEnv::current()->record_failure("CodeCache is full");
 2272     return 0;  // CodeBuffer::expand failed
 2273   }
 2274   int offset = __ offset();
 2275 
 2276   __ adr(lr, __ pc());
 2277   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2278 
 2279   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2280   __ end_a_stub();
 2281   return offset;
 2282 }
 2283 
 2284 // REQUIRED MATCHER CODE
 2285 
 2286 //=============================================================================
 2287 
 2288 const bool Matcher::match_rule_supported(int opcode) {
 2289   if (!has_match_rule(opcode))
 2290     return false;
 2291 
 2292   bool ret_value = true;
 2293   switch (opcode) {
 2294     case Op_OnSpinWait:
 2295       return VM_Version::supports_on_spin_wait();
 2296     case Op_CacheWB:
 2297     case Op_CacheWBPreSync:
 2298     case Op_CacheWBPostSync:
 2299       if (!VM_Version::supports_data_cache_line_flush()) {
 2300         ret_value = false;
 2301       }
 2302       break;
 2303     case Op_ExpandBits:
 2304     case Op_CompressBits:
 2305       if (!(UseSVE > 1 && VM_Version::supports_svebitperm())) {
 2306         ret_value = false;
 2307       }
 2308       break;
 2309   }
 2310 
 2311   return ret_value; // Per default match rules are supported.
 2312 }
 2313 
 2314 const RegMask* Matcher::predicate_reg_mask(void) {
 2315   return &_PR_REG_mask;
 2316 }
 2317 
 2318 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2319   return new TypeVectMask(elemTy, length);
 2320 }
 2321 
 2322 // Vector calling convention not yet implemented.
 2323 const bool Matcher::supports_vector_calling_convention(void) {
 2324   return false;
 2325 }
 2326 
 2327 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2328   Unimplemented();
 2329   return OptoRegPair(0, 0);
 2330 }
 2331 
 2332 // Is this branch offset short enough that a short branch can be used?
 2333 //
 2334 // NOTE: If the platform does not provide any short branch variants, then
 2335 //       this method should return false for offset 0.
 2336 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2337   // The passed offset is relative to address of the branch.
 2338 
 2339   return (-32768 <= offset && offset < 32768);
 2340 }
 2341 
 2342 // Vector width in bytes.
 2343 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2344   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2345   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2346   // Minimum 2 values in vector
 2347   if (size < 2*type2aelembytes(bt)) size = 0;
 2348   // But never < 4
 2349   if (size < 4) size = 0;
 2350   return size;
 2351 }
 2352 
 2353 // Limits on vector size (number of elements) loaded into vector.
 2354 const int Matcher::max_vector_size(const BasicType bt) {
 2355   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2356 }
 2357 
 2358 const int Matcher::min_vector_size(const BasicType bt) {
 2359   int max_size = max_vector_size(bt);
 2360   // Limit the min vector size to 8 bytes.
 2361   int size = 8 / type2aelembytes(bt);
 2362   if (bt == T_BYTE) {
 2363     // To support vector api shuffle/rearrange.
 2364     size = 4;
 2365   } else if (bt == T_BOOLEAN) {
 2366     // To support vector api load/store mask.
 2367     size = 2;
 2368   }
 2369   if (size < 2) size = 2;
 2370   return MIN2(size, max_size);
 2371 }
 2372 
 2373 const int Matcher::superword_max_vector_size(const BasicType bt) {
 2374   return Matcher::max_vector_size(bt);
 2375 }
 2376 
 2377 // Actual max scalable vector register length.
 2378 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2379   return Matcher::max_vector_size(bt);
 2380 }
 2381 
 2382 // Vector ideal reg.
 2383 const uint Matcher::vector_ideal_reg(int len) {
 2384   if (UseSVE > 0 && 16 < len && len <= 256) {
 2385     return Op_VecA;
 2386   }
 2387   switch(len) {
 2388     // For 16-bit/32-bit mask vector, reuse VecD.
 2389     case  2:
 2390     case  4:
 2391     case  8: return Op_VecD;
 2392     case 16: return Op_VecX;
 2393   }
 2394   ShouldNotReachHere();
 2395   return 0;
 2396 }
 2397 
 2398 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2399   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2400   switch (ideal_reg) {
 2401     case Op_VecA: return new vecAOper();
 2402     case Op_VecD: return new vecDOper();
 2403     case Op_VecX: return new vecXOper();
 2404   }
 2405   ShouldNotReachHere();
 2406   return NULL;
 2407 }
 2408 
 2409 bool Matcher::is_reg2reg_move(MachNode* m) {
 2410   return false;
 2411 }
 2412 
 2413 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2414   return opnd->opcode() == VREG;
 2415 }
 2416 
 2417 // Return whether or not this register is ever used as an argument.
 2418 // This function is used on startup to build the trampoline stubs in
 2419 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2420 // call in the trampoline, and arguments in those registers not be
 2421 // available to the callee.
 2422 bool Matcher::can_be_java_arg(int reg)
 2423 {
 2424   return
 2425     reg ==  R0_num || reg == R0_H_num ||
 2426     reg ==  R1_num || reg == R1_H_num ||
 2427     reg ==  R2_num || reg == R2_H_num ||
 2428     reg ==  R3_num || reg == R3_H_num ||
 2429     reg ==  R4_num || reg == R4_H_num ||
 2430     reg ==  R5_num || reg == R5_H_num ||
 2431     reg ==  R6_num || reg == R6_H_num ||
 2432     reg ==  R7_num || reg == R7_H_num ||
 2433     reg ==  V0_num || reg == V0_H_num ||
 2434     reg ==  V1_num || reg == V1_H_num ||
 2435     reg ==  V2_num || reg == V2_H_num ||
 2436     reg ==  V3_num || reg == V3_H_num ||
 2437     reg ==  V4_num || reg == V4_H_num ||
 2438     reg ==  V5_num || reg == V5_H_num ||
 2439     reg ==  V6_num || reg == V6_H_num ||
 2440     reg ==  V7_num || reg == V7_H_num;
 2441 }
 2442 
 2443 bool Matcher::is_spillable_arg(int reg)
 2444 {
 2445   return can_be_java_arg(reg);
 2446 }
 2447 
 2448 uint Matcher::int_pressure_limit()
 2449 {
 2450   // JDK-8183543: When taking the number of available registers as int
 2451   // register pressure threshold, the jtreg test:
 2452   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2453   // failed due to C2 compilation failure with
 2454   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2455   //
 2456   // A derived pointer is live at CallNode and then is flagged by RA
 2457   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2458   // derived pointers and lastly fail to spill after reaching maximum
 2459   // number of iterations. Lowering the default pressure threshold to
 2460   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2461   // a high register pressure area of the code so that split_DEF can
 2462   // generate DefinitionSpillCopy for the derived pointer.
 2463   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2464   if (!PreserveFramePointer) {
 2465     // When PreserveFramePointer is off, frame pointer is allocatable,
 2466     // but different from other SOC registers, it is excluded from
 2467     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2468     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2469     // See check_pressure_at_fatproj().
 2470     default_int_pressure_threshold--;
 2471   }
 2472   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2473 }
 2474 
 2475 uint Matcher::float_pressure_limit()
 2476 {
 2477   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2478   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2479 }
 2480 
 2481 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2482   return false;
 2483 }
 2484 
 2485 RegMask Matcher::divI_proj_mask() {
 2486   ShouldNotReachHere();
 2487   return RegMask();
 2488 }
 2489 
 2490 // Register for MODI projection of divmodI.
 2491 RegMask Matcher::modI_proj_mask() {
 2492   ShouldNotReachHere();
 2493   return RegMask();
 2494 }
 2495 
 2496 // Register for DIVL projection of divmodL.
 2497 RegMask Matcher::divL_proj_mask() {
 2498   ShouldNotReachHere();
 2499   return RegMask();
 2500 }
 2501 
 2502 // Register for MODL projection of divmodL.
 2503 RegMask Matcher::modL_proj_mask() {
 2504   ShouldNotReachHere();
 2505   return RegMask();
 2506 }
 2507 
 2508 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2509   return FP_REG_mask();
 2510 }
 2511 
 2512 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2513   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2514     Node* u = addp->fast_out(i);
 2515     if (u->is_LoadStore()) {
 2516       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2517       // instructions) only take register indirect as an operand, so
 2518       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2519       // must fail.
 2520       return false;
 2521     }
 2522     if (u->is_Mem()) {
 2523       int opsize = u->as_Mem()->memory_size();
 2524       assert(opsize > 0, "unexpected memory operand size");
 2525       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2526         return false;
 2527       }
 2528     }
 2529   }
 2530   return true;
 2531 }
 2532 
 2533 // Convert BootTest condition to Assembler condition.
 2534 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2535 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2536   Assembler::Condition result;
 2537   switch(cond) {
 2538     case BoolTest::eq:
 2539       result = Assembler::EQ; break;
 2540     case BoolTest::ne:
 2541       result = Assembler::NE; break;
 2542     case BoolTest::le:
 2543       result = Assembler::LE; break;
 2544     case BoolTest::ge:
 2545       result = Assembler::GE; break;
 2546     case BoolTest::lt:
 2547       result = Assembler::LT; break;
 2548     case BoolTest::gt:
 2549       result = Assembler::GT; break;
 2550     case BoolTest::ule:
 2551       result = Assembler::LS; break;
 2552     case BoolTest::uge:
 2553       result = Assembler::HS; break;
 2554     case BoolTest::ult:
 2555       result = Assembler::LO; break;
 2556     case BoolTest::ugt:
 2557       result = Assembler::HI; break;
 2558     case BoolTest::overflow:
 2559       result = Assembler::VS; break;
 2560     case BoolTest::no_overflow:
 2561       result = Assembler::VC; break;
 2562     default:
 2563       ShouldNotReachHere();
 2564       return Assembler::Condition(-1);
 2565   }
 2566 
 2567   // Check conversion
 2568   if (cond & BoolTest::unsigned_compare) {
 2569     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2570   } else {
 2571     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2572   }
 2573 
 2574   return result;
 2575 }
 2576 
 2577 // Binary src (Replicate con)
 2578 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2579   if (n == NULL || m == NULL) {
 2580     return false;
 2581   }
 2582 
 2583   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2584     return false;
 2585   }
 2586 
 2587   Node* imm_node = m->in(1);
 2588   if (!imm_node->is_Con()) {
 2589     return false;
 2590   }
 2591 
 2592   const Type* t = imm_node->bottom_type();
 2593   if (!(t->isa_int() || t->isa_long())) {
 2594     return false;
 2595   }
 2596 
 2597   switch (n->Opcode()) {
 2598   case Op_AndV:
 2599   case Op_OrV:
 2600   case Op_XorV: {
 2601     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2602     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2603     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2604   }
 2605   case Op_AddVB:
 2606     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2607   case Op_AddVS:
 2608   case Op_AddVI:
 2609     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2610   case Op_AddVL:
 2611     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2612   default:
 2613     return false;
 2614   }
 2615 }
 2616 
 2617 // (XorV src (Replicate m1))
 2618 // (XorVMask src (MaskAll m1))
 2619 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2620   if (n != NULL && m != NULL) {
 2621     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2622            VectorNode::is_all_ones_vector(m);
 2623   }
 2624   return false;
 2625 }
 2626 
 2627 // Should the matcher clone input 'm' of node 'n'?
 2628 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2629   if (is_vshift_con_pattern(n, m) ||
 2630       is_vector_bitwise_not_pattern(n, m) ||
 2631       is_valid_sve_arith_imm_pattern(n, m)) {
 2632     mstack.push(m, Visit);
 2633     return true;
 2634   }
 2635   return false;
 2636 }
 2637 
 2638 // Should the Matcher clone shifts on addressing modes, expecting them
 2639 // to be subsumed into complex addressing expressions or compute them
 2640 // into registers?
 2641 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2642   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2643     return true;
 2644   }
 2645 
 2646   Node *off = m->in(AddPNode::Offset);
 2647   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2648       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2649       // Are there other uses besides address expressions?
 2650       !is_visited(off)) {
 2651     address_visited.set(off->_idx); // Flag as address_visited
 2652     mstack.push(off->in(2), Visit);
 2653     Node *conv = off->in(1);
 2654     if (conv->Opcode() == Op_ConvI2L &&
 2655         // Are there other uses besides address expressions?
 2656         !is_visited(conv)) {
 2657       address_visited.set(conv->_idx); // Flag as address_visited
 2658       mstack.push(conv->in(1), Pre_Visit);
 2659     } else {
 2660       mstack.push(conv, Pre_Visit);
 2661     }
 2662     address_visited.test_set(m->_idx); // Flag as address_visited
 2663     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2664     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2665     return true;
 2666   } else if (off->Opcode() == Op_ConvI2L &&
 2667              // Are there other uses besides address expressions?
 2668              !is_visited(off)) {
 2669     address_visited.test_set(m->_idx); // Flag as address_visited
 2670     address_visited.set(off->_idx); // Flag as address_visited
 2671     mstack.push(off->in(1), Pre_Visit);
 2672     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2673     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2674     return true;
 2675   }
 2676   return false;
 2677 }
 2678 
 2679 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2680   C2_MacroAssembler _masm(&cbuf);                                       \
 2681   {                                                                     \
 2682     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2683     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2684     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2685     __ INSN(REG, as_Register(BASE));                                    \
 2686   }
 2687 
 2688 
 2689 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2690   {
 2691     Address::extend scale;
 2692 
 2693     // Hooboy, this is fugly.  We need a way to communicate to the
 2694     // encoder that the index needs to be sign extended, so we have to
 2695     // enumerate all the cases.
 2696     switch (opcode) {
 2697     case INDINDEXSCALEDI2L:
 2698     case INDINDEXSCALEDI2LN:
 2699     case INDINDEXI2L:
 2700     case INDINDEXI2LN:
 2701       scale = Address::sxtw(size);
 2702       break;
 2703     default:
 2704       scale = Address::lsl(size);
 2705     }
 2706 
 2707     if (index == -1) {
 2708       return Address(base, disp);
 2709     } else {
 2710       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2711       return Address(base, as_Register(index), scale);
 2712     }
 2713   }
 2714 
 2715 
 2716 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2717 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2718 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2719 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2720                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2721 
 2722   // Used for all non-volatile memory accesses.  The use of
 2723   // $mem->opcode() to discover whether this pattern uses sign-extended
 2724   // offsets is something of a kludge.
 2725   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2726                         Register reg, int opcode,
 2727                         Register base, int index, int scale, int disp,
 2728                         int size_in_memory)
 2729   {
 2730     Address addr = mem2address(opcode, base, index, scale, disp);
 2731     if (addr.getMode() == Address::base_plus_offset) {
 2732       /* If we get an out-of-range offset it is a bug in the compiler,
 2733          so we assert here. */
 2734       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2735              "c2 compiler bug");
 2736       /* Fix up any out-of-range offsets. */
 2737       assert_different_registers(rscratch1, base);
 2738       assert_different_registers(rscratch1, reg);
 2739       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2740     }
 2741     (masm.*insn)(reg, addr);
 2742   }
 2743 
 2744   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2745                         FloatRegister reg, int opcode,
 2746                         Register base, int index, int size, int disp,
 2747                         int size_in_memory)
 2748   {
 2749     Address::extend scale;
 2750 
 2751     switch (opcode) {
 2752     case INDINDEXSCALEDI2L:
 2753     case INDINDEXSCALEDI2LN:
 2754       scale = Address::sxtw(size);
 2755       break;
 2756     default:
 2757       scale = Address::lsl(size);
 2758     }
 2759 
 2760     if (index == -1) {
 2761       /* If we get an out-of-range offset it is a bug in the compiler,
 2762          so we assert here. */
 2763       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2764       /* Fix up any out-of-range offsets. */
 2765       assert_different_registers(rscratch1, base);
 2766       Address addr = Address(base, disp);
 2767       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2768       (masm.*insn)(reg, addr);
 2769     } else {
 2770       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2771       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2772     }
 2773   }
 2774 
 2775   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2776                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2777                         int opcode, Register base, int index, int size, int disp)
 2778   {
 2779     if (index == -1) {
 2780       (masm.*insn)(reg, T, Address(base, disp));
 2781     } else {
 2782       assert(disp == 0, "unsupported address mode");
 2783       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2784     }
 2785   }
 2786 
 2787 %}
 2788 
 2789 
 2790 
 2791 //----------ENCODING BLOCK-----------------------------------------------------
 2792 // This block specifies the encoding classes used by the compiler to
 2793 // output byte streams.  Encoding classes are parameterized macros
 2794 // used by Machine Instruction Nodes in order to generate the bit
 2795 // encoding of the instruction.  Operands specify their base encoding
 2796 // interface with the interface keyword.  There are currently
 2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2798 // COND_INTER.  REG_INTER causes an operand to generate a function
 2799 // which returns its register number when queried.  CONST_INTER causes
 2800 // an operand to generate a function which returns the value of the
 2801 // constant when queried.  MEMORY_INTER causes an operand to generate
 2802 // four functions which return the Base Register, the Index Register,
 2803 // the Scale Value, and the Offset Value of the operand when queried.
 2804 // COND_INTER causes an operand to generate six functions which return
 2805 // the encoding code (ie - encoding bits for the instruction)
 2806 // associated with each basic boolean condition for a conditional
 2807 // instruction.
 2808 //
 2809 // Instructions specify two basic values for encoding.  Again, a
 2810 // function is available to check if the constant displacement is an
 2811 // oop. They use the ins_encode keyword to specify their encoding
 2812 // classes (which must be a sequence of enc_class names, and their
 2813 // parameters, specified in the encoding block), and they use the
 2814 // opcode keyword to specify, in order, their primary, secondary, and
 2815 // tertiary opcode.  Only the opcode sections which a particular
 2816 // instruction needs for encoding need to be specified.
 2817 encode %{
 2818   // Build emit functions for each basic byte or larger field in the
 2819   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2820   // from C++ code in the enc_class source block.  Emit functions will
 2821   // live in the main source block for now.  In future, we can
 2822   // generalize this by adding a syntax that specifies the sizes of
 2823   // fields in an order, so that the adlc can build the emit functions
 2824   // automagically
 2825 
 2826   // catch all for unimplemented encodings
 2827   enc_class enc_unimplemented %{
 2828     C2_MacroAssembler _masm(&cbuf);
 2829     __ unimplemented("C2 catch all");
 2830   %}
 2831 
 2832   // BEGIN Non-volatile memory access
 2833 
 2834   // This encoding class is generated automatically from ad_encode.m4.
 2835   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2836   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2837     Register dst_reg = as_Register($dst$$reg);
 2838     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2839                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2840   %}
 2841 
 2842   // This encoding class is generated automatically from ad_encode.m4.
 2843   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2844   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2845     Register dst_reg = as_Register($dst$$reg);
 2846     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2847                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2848   %}
 2849 
 2850   // This encoding class is generated automatically from ad_encode.m4.
 2851   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2852   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2853     Register dst_reg = as_Register($dst$$reg);
 2854     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2855                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2856   %}
 2857 
 2858   // This encoding class is generated automatically from ad_encode.m4.
 2859   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2860   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2861     Register dst_reg = as_Register($dst$$reg);
 2862     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2863                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2864   %}
 2865 
 2866   // This encoding class is generated automatically from ad_encode.m4.
 2867   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2868   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2869     Register dst_reg = as_Register($dst$$reg);
 2870     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2871                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2872   %}
 2873 
 2874   // This encoding class is generated automatically from ad_encode.m4.
 2875   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2876   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2877     Register dst_reg = as_Register($dst$$reg);
 2878     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2879                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2880   %}
 2881 
 2882   // This encoding class is generated automatically from ad_encode.m4.
 2883   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2884   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2885     Register dst_reg = as_Register($dst$$reg);
 2886     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2887                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2888   %}
 2889 
 2890   // This encoding class is generated automatically from ad_encode.m4.
 2891   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2892   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2893     Register dst_reg = as_Register($dst$$reg);
 2894     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2895                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2896   %}
 2897 
 2898   // This encoding class is generated automatically from ad_encode.m4.
 2899   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2900   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2901     Register dst_reg = as_Register($dst$$reg);
 2902     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2903                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2904   %}
 2905 
 2906   // This encoding class is generated automatically from ad_encode.m4.
 2907   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2908   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2909     Register dst_reg = as_Register($dst$$reg);
 2910     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2911                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2912   %}
 2913 
 2914   // This encoding class is generated automatically from ad_encode.m4.
 2915   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2916   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2917     Register dst_reg = as_Register($dst$$reg);
 2918     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2919                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2920   %}
 2921 
 2922   // This encoding class is generated automatically from ad_encode.m4.
 2923   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2924   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2925     Register dst_reg = as_Register($dst$$reg);
 2926     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2927                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2928   %}
 2929 
 2930   // This encoding class is generated automatically from ad_encode.m4.
 2931   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2932   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2933     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2934     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2935                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2936   %}
 2937 
 2938   // This encoding class is generated automatically from ad_encode.m4.
 2939   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2940   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2941     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2942     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2943                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2944   %}
 2945 
 2946   // This encoding class is generated automatically from ad_encode.m4.
 2947   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2948   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2949     Register src_reg = as_Register($src$$reg);
 2950     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2951                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2952   %}
 2953 
 2954   // This encoding class is generated automatically from ad_encode.m4.
 2955   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2956   enc_class aarch64_enc_strb0(memory1 mem) %{
 2957     C2_MacroAssembler _masm(&cbuf);
 2958     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2965     Register src_reg = as_Register($src$$reg);
 2966     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_strh0(memory2 mem) %{
 2973     C2_MacroAssembler _masm(&cbuf);
 2974     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2975                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2976   %}
 2977 
 2978   // This encoding class is generated automatically from ad_encode.m4.
 2979   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2980   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2981     Register src_reg = as_Register($src$$reg);
 2982     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2983                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2984   %}
 2985 
 2986   // This encoding class is generated automatically from ad_encode.m4.
 2987   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2988   enc_class aarch64_enc_strw0(memory4 mem) %{
 2989     C2_MacroAssembler _masm(&cbuf);
 2990     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2991                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2992   %}
 2993 
 2994   // This encoding class is generated automatically from ad_encode.m4.
 2995   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2996   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2997     Register src_reg = as_Register($src$$reg);
 2998     // we sometimes get asked to store the stack pointer into the
 2999     // current thread -- we cannot do that directly on AArch64
 3000     if (src_reg == r31_sp) {
 3001       C2_MacroAssembler _masm(&cbuf);
 3002       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3003       __ mov(rscratch2, sp);
 3004       src_reg = rscratch2;
 3005     }
 3006     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 3007                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3008   %}
 3009 
 3010   // This encoding class is generated automatically from ad_encode.m4.
 3011   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3012   enc_class aarch64_enc_str0(memory8 mem) %{
 3013     C2_MacroAssembler _masm(&cbuf);
 3014     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 3015                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3016   %}
 3017 
 3018   // This encoding class is generated automatically from ad_encode.m4.
 3019   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3020   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3021     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3022     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3029     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3030     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 3031                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3032   %}
 3033 
 3034   // This encoding class is generated automatically from ad_encode.m4.
 3035   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3036   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3037       C2_MacroAssembler _masm(&cbuf);
 3038       __ membar(Assembler::StoreStore);
 3039       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3040                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3041   %}
 3042 
 3043   // END Non-volatile memory access
 3044 
 3045   // Vector loads and stores
 3046   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3047     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3048     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3049        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3050   %}
 3051 
 3052   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3053     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3054     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3055        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3056   %}
 3057 
 3058   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3059     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3060     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3061        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3062   %}
 3063 
 3064   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3065     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3066     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3067        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3068   %}
 3069 
 3070   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3071     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3072     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3073        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3074   %}
 3075 
 3076   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3077     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3078     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3079        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3080   %}
 3081 
 3082   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3083     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3084     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3085        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3086   %}
 3087 
 3088   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3089     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3090     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3091        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3092   %}
 3093 
 3094   // volatile loads and stores
 3095 
 3096   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3097     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3098                  rscratch1, stlrb);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_stlrb0(memory mem) %{
 3102     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3103                  rscratch1, stlrb);
 3104   %}
 3105 
 3106   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3107     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3108                  rscratch1, stlrh);
 3109   %}
 3110 
 3111   enc_class aarch64_enc_stlrh0(memory mem) %{
 3112     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3113                  rscratch1, stlrh);
 3114   %}
 3115 
 3116   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3117     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3118                  rscratch1, stlrw);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_stlrw0(memory mem) %{
 3122     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3123                  rscratch1, stlrw);
 3124   %}
 3125 
 3126   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3127     Register dst_reg = as_Register($dst$$reg);
 3128     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3129              rscratch1, ldarb);
 3130     __ sxtbw(dst_reg, dst_reg);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3134     Register dst_reg = as_Register($dst$$reg);
 3135     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136              rscratch1, ldarb);
 3137     __ sxtb(dst_reg, dst_reg);
 3138   %}
 3139 
 3140   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3141     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3142              rscratch1, ldarb);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3146     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3147              rscratch1, ldarb);
 3148   %}
 3149 
 3150   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3151     Register dst_reg = as_Register($dst$$reg);
 3152     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3153              rscratch1, ldarh);
 3154     __ sxthw(dst_reg, dst_reg);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3158     Register dst_reg = as_Register($dst$$reg);
 3159     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160              rscratch1, ldarh);
 3161     __ sxth(dst_reg, dst_reg);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldarh);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3170     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171              rscratch1, ldarh);
 3172   %}
 3173 
 3174   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3175     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3176              rscratch1, ldarw);
 3177   %}
 3178 
 3179   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3180     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3181              rscratch1, ldarw);
 3182   %}
 3183 
 3184   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3185     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3186              rscratch1, ldar);
 3187   %}
 3188 
 3189   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3190     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3191              rscratch1, ldarw);
 3192     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3193   %}
 3194 
 3195   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3196     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3197              rscratch1, ldar);
 3198     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3199   %}
 3200 
 3201   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3202     Register src_reg = as_Register($src$$reg);
 3203     // we sometimes get asked to store the stack pointer into the
 3204     // current thread -- we cannot do that directly on AArch64
 3205     if (src_reg == r31_sp) {
 3206       C2_MacroAssembler _masm(&cbuf);
 3207       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3208       __ mov(rscratch2, sp);
 3209       src_reg = rscratch2;
 3210     }
 3211     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3212                  rscratch1, stlr);
 3213   %}
 3214 
 3215   enc_class aarch64_enc_stlr0(memory mem) %{
 3216     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217                  rscratch1, stlr);
 3218   %}
 3219 
 3220   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3221     {
 3222       C2_MacroAssembler _masm(&cbuf);
 3223       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3224       __ fmovs(rscratch2, src_reg);
 3225     }
 3226     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3227                  rscratch1, stlrw);
 3228   %}
 3229 
 3230   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3231     {
 3232       C2_MacroAssembler _masm(&cbuf);
 3233       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3234       __ fmovd(rscratch2, src_reg);
 3235     }
 3236     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3237                  rscratch1, stlr);
 3238   %}
 3239 
 3240   // synchronized read/update encodings
 3241 
 3242   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3243     C2_MacroAssembler _masm(&cbuf);
 3244     Register dst_reg = as_Register($dst$$reg);
 3245     Register base = as_Register($mem$$base);
 3246     int index = $mem$$index;
 3247     int scale = $mem$$scale;
 3248     int disp = $mem$$disp;
 3249     if (index == -1) {
 3250        if (disp != 0) {
 3251         __ lea(rscratch1, Address(base, disp));
 3252         __ ldaxr(dst_reg, rscratch1);
 3253       } else {
 3254         // TODO
 3255         // should we ever get anything other than this case?
 3256         __ ldaxr(dst_reg, base);
 3257       }
 3258     } else {
 3259       Register index_reg = as_Register(index);
 3260       if (disp == 0) {
 3261         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3262         __ ldaxr(dst_reg, rscratch1);
 3263       } else {
 3264         __ lea(rscratch1, Address(base, disp));
 3265         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3266         __ ldaxr(dst_reg, rscratch1);
 3267       }
 3268     }
 3269   %}
 3270 
 3271   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3272     C2_MacroAssembler _masm(&cbuf);
 3273     Register src_reg = as_Register($src$$reg);
 3274     Register base = as_Register($mem$$base);
 3275     int index = $mem$$index;
 3276     int scale = $mem$$scale;
 3277     int disp = $mem$$disp;
 3278     if (index == -1) {
 3279        if (disp != 0) {
 3280         __ lea(rscratch2, Address(base, disp));
 3281         __ stlxr(rscratch1, src_reg, rscratch2);
 3282       } else {
 3283         // TODO
 3284         // should we ever get anything other than this case?
 3285         __ stlxr(rscratch1, src_reg, base);
 3286       }
 3287     } else {
 3288       Register index_reg = as_Register(index);
 3289       if (disp == 0) {
 3290         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3291         __ stlxr(rscratch1, src_reg, rscratch2);
 3292       } else {
 3293         __ lea(rscratch2, Address(base, disp));
 3294         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3295         __ stlxr(rscratch1, src_reg, rscratch2);
 3296       }
 3297     }
 3298     __ cmpw(rscratch1, zr);
 3299   %}
 3300 
 3301   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3302     C2_MacroAssembler _masm(&cbuf);
 3303     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3304     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3305                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3306                /*weak*/ false, noreg);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3310     C2_MacroAssembler _masm(&cbuf);
 3311     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3312     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3313                Assembler::word, /*acquire*/ false, /*release*/ true,
 3314                /*weak*/ false, noreg);
 3315   %}
 3316 
 3317   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3318     C2_MacroAssembler _masm(&cbuf);
 3319     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3320     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3321                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3322                /*weak*/ false, noreg);
 3323   %}
 3324 
 3325   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3326     C2_MacroAssembler _masm(&cbuf);
 3327     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3328     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3329                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3330                /*weak*/ false, noreg);
 3331   %}
 3332 
 3333 
 3334   // The only difference between aarch64_enc_cmpxchg and
 3335   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3336   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3337   // lock.
 3338   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3339     C2_MacroAssembler _masm(&cbuf);
 3340     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3341     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3342                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3343                /*weak*/ false, noreg);
 3344   %}
 3345 
 3346   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3347     C2_MacroAssembler _masm(&cbuf);
 3348     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3349     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3350                Assembler::word, /*acquire*/ true, /*release*/ true,
 3351                /*weak*/ false, noreg);
 3352   %}
 3353 
 3354   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3355     C2_MacroAssembler _masm(&cbuf);
 3356     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3357     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3358                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3359                /*weak*/ false, noreg);
 3360   %}
 3361 
 3362   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3363     C2_MacroAssembler _masm(&cbuf);
 3364     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3365     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3366                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3367                /*weak*/ false, noreg);
 3368   %}
 3369 
 3370   // auxiliary used for CompareAndSwapX to set result register
 3371   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3372     C2_MacroAssembler _masm(&cbuf);
 3373     Register res_reg = as_Register($res$$reg);
 3374     __ cset(res_reg, Assembler::EQ);
 3375   %}
 3376 
 3377   // prefetch encodings
 3378 
 3379   enc_class aarch64_enc_prefetchw(memory mem) %{
 3380     C2_MacroAssembler _masm(&cbuf);
 3381     Register base = as_Register($mem$$base);
 3382     int index = $mem$$index;
 3383     int scale = $mem$$scale;
 3384     int disp = $mem$$disp;
 3385     if (index == -1) {
 3386       __ prfm(Address(base, disp), PSTL1KEEP);
 3387     } else {
 3388       Register index_reg = as_Register(index);
 3389       if (disp == 0) {
 3390         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3391       } else {
 3392         __ lea(rscratch1, Address(base, disp));
 3393 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3394       }
 3395     }
 3396   %}
 3397 
 3398   /// mov envcodings
 3399 
 3400   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3401     C2_MacroAssembler _masm(&cbuf);
 3402     uint32_t con = (uint32_t)$src$$constant;
 3403     Register dst_reg = as_Register($dst$$reg);
 3404     if (con == 0) {
 3405       __ movw(dst_reg, zr);
 3406     } else {
 3407       __ movw(dst_reg, con);
 3408     }
 3409   %}
 3410 
 3411   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3412     C2_MacroAssembler _masm(&cbuf);
 3413     Register dst_reg = as_Register($dst$$reg);
 3414     uint64_t con = (uint64_t)$src$$constant;
 3415     if (con == 0) {
 3416       __ mov(dst_reg, zr);
 3417     } else {
 3418       __ mov(dst_reg, con);
 3419     }
 3420   %}
 3421 
 3422   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3423     C2_MacroAssembler _masm(&cbuf);
 3424     Register dst_reg = as_Register($dst$$reg);
 3425     address con = (address)$src$$constant;
 3426     if (con == NULL || con == (address)1) {
 3427       ShouldNotReachHere();
 3428     } else {
 3429       relocInfo::relocType rtype = $src->constant_reloc();
 3430       if (rtype == relocInfo::oop_type) {
 3431         __ movoop(dst_reg, (jobject)con);
 3432       } else if (rtype == relocInfo::metadata_type) {
 3433         __ mov_metadata(dst_reg, (Metadata*)con);
 3434       } else {
 3435         assert(rtype == relocInfo::none, "unexpected reloc type");
 3436         if (! __ is_valid_AArch64_address(con) ||
 3437             con < (address)(uintptr_t)os::vm_page_size()) {
 3438           __ mov(dst_reg, con);
 3439         } else {
 3440           uint64_t offset;
 3441           __ adrp(dst_reg, con, offset);
 3442           __ add(dst_reg, dst_reg, offset);
 3443         }
 3444       }
 3445     }
 3446   %}
 3447 
 3448   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3449     C2_MacroAssembler _masm(&cbuf);
 3450     Register dst_reg = as_Register($dst$$reg);
 3451     __ mov(dst_reg, zr);
 3452   %}
 3453 
 3454   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3455     C2_MacroAssembler _masm(&cbuf);
 3456     Register dst_reg = as_Register($dst$$reg);
 3457     __ mov(dst_reg, (uint64_t)1);
 3458   %}
 3459 
 3460   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3461     C2_MacroAssembler _masm(&cbuf);
 3462     __ load_byte_map_base($dst$$Register);
 3463   %}
 3464 
 3465   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3466     C2_MacroAssembler _masm(&cbuf);
 3467     Register dst_reg = as_Register($dst$$reg);
 3468     address con = (address)$src$$constant;
 3469     if (con == NULL) {
 3470       ShouldNotReachHere();
 3471     } else {
 3472       relocInfo::relocType rtype = $src->constant_reloc();
 3473       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3474       __ set_narrow_oop(dst_reg, (jobject)con);
 3475     }
 3476   %}
 3477 
 3478   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3479     C2_MacroAssembler _masm(&cbuf);
 3480     Register dst_reg = as_Register($dst$$reg);
 3481     __ mov(dst_reg, zr);
 3482   %}
 3483 
 3484   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3485     C2_MacroAssembler _masm(&cbuf);
 3486     Register dst_reg = as_Register($dst$$reg);
 3487     address con = (address)$src$$constant;
 3488     if (con == NULL) {
 3489       ShouldNotReachHere();
 3490     } else {
 3491       relocInfo::relocType rtype = $src->constant_reloc();
 3492       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3493       __ set_narrow_klass(dst_reg, (Klass *)con);
 3494     }
 3495   %}
 3496 
 3497   // arithmetic encodings
 3498 
 3499   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3500     C2_MacroAssembler _masm(&cbuf);
 3501     Register dst_reg = as_Register($dst$$reg);
 3502     Register src_reg = as_Register($src1$$reg);
 3503     int32_t con = (int32_t)$src2$$constant;
 3504     // add has primary == 0, subtract has primary == 1
 3505     if ($primary) { con = -con; }
 3506     if (con < 0) {
 3507       __ subw(dst_reg, src_reg, -con);
 3508     } else {
 3509       __ addw(dst_reg, src_reg, con);
 3510     }
 3511   %}
 3512 
 3513   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3514     C2_MacroAssembler _masm(&cbuf);
 3515     Register dst_reg = as_Register($dst$$reg);
 3516     Register src_reg = as_Register($src1$$reg);
 3517     int32_t con = (int32_t)$src2$$constant;
 3518     // add has primary == 0, subtract has primary == 1
 3519     if ($primary) { con = -con; }
 3520     if (con < 0) {
 3521       __ sub(dst_reg, src_reg, -con);
 3522     } else {
 3523       __ add(dst_reg, src_reg, con);
 3524     }
 3525   %}
 3526 
 3527   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3528     C2_MacroAssembler _masm(&cbuf);
 3529    Register dst_reg = as_Register($dst$$reg);
 3530    Register src1_reg = as_Register($src1$$reg);
 3531    Register src2_reg = as_Register($src2$$reg);
 3532     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3533   %}
 3534 
 3535   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537    Register dst_reg = as_Register($dst$$reg);
 3538    Register src1_reg = as_Register($src1$$reg);
 3539    Register src2_reg = as_Register($src2$$reg);
 3540     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3541   %}
 3542 
 3543   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3544     C2_MacroAssembler _masm(&cbuf);
 3545    Register dst_reg = as_Register($dst$$reg);
 3546    Register src1_reg = as_Register($src1$$reg);
 3547    Register src2_reg = as_Register($src2$$reg);
 3548     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3549   %}
 3550 
 3551   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3552     C2_MacroAssembler _masm(&cbuf);
 3553    Register dst_reg = as_Register($dst$$reg);
 3554    Register src1_reg = as_Register($src1$$reg);
 3555    Register src2_reg = as_Register($src2$$reg);
 3556     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3557   %}
 3558 
 3559   // compare instruction encodings
 3560 
 3561   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3562     C2_MacroAssembler _masm(&cbuf);
 3563     Register reg1 = as_Register($src1$$reg);
 3564     Register reg2 = as_Register($src2$$reg);
 3565     __ cmpw(reg1, reg2);
 3566   %}
 3567 
 3568   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3569     C2_MacroAssembler _masm(&cbuf);
 3570     Register reg = as_Register($src1$$reg);
 3571     int32_t val = $src2$$constant;
 3572     if (val >= 0) {
 3573       __ subsw(zr, reg, val);
 3574     } else {
 3575       __ addsw(zr, reg, -val);
 3576     }
 3577   %}
 3578 
 3579   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3580     C2_MacroAssembler _masm(&cbuf);
 3581     Register reg1 = as_Register($src1$$reg);
 3582     uint32_t val = (uint32_t)$src2$$constant;
 3583     __ movw(rscratch1, val);
 3584     __ cmpw(reg1, rscratch1);
 3585   %}
 3586 
 3587   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3588     C2_MacroAssembler _masm(&cbuf);
 3589     Register reg1 = as_Register($src1$$reg);
 3590     Register reg2 = as_Register($src2$$reg);
 3591     __ cmp(reg1, reg2);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3595     C2_MacroAssembler _masm(&cbuf);
 3596     Register reg = as_Register($src1$$reg);
 3597     int64_t val = $src2$$constant;
 3598     if (val >= 0) {
 3599       __ subs(zr, reg, val);
 3600     } else if (val != -val) {
 3601       __ adds(zr, reg, -val);
 3602     } else {
 3603     // aargh, Long.MIN_VALUE is a special case
 3604       __ orr(rscratch1, zr, (uint64_t)val);
 3605       __ subs(zr, reg, rscratch1);
 3606     }
 3607   %}
 3608 
 3609   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3610     C2_MacroAssembler _masm(&cbuf);
 3611     Register reg1 = as_Register($src1$$reg);
 3612     uint64_t val = (uint64_t)$src2$$constant;
 3613     __ mov(rscratch1, val);
 3614     __ cmp(reg1, rscratch1);
 3615   %}
 3616 
 3617   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3618     C2_MacroAssembler _masm(&cbuf);
 3619     Register reg1 = as_Register($src1$$reg);
 3620     Register reg2 = as_Register($src2$$reg);
 3621     __ cmp(reg1, reg2);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3625     C2_MacroAssembler _masm(&cbuf);
 3626     Register reg1 = as_Register($src1$$reg);
 3627     Register reg2 = as_Register($src2$$reg);
 3628     __ cmpw(reg1, reg2);
 3629   %}
 3630 
 3631   enc_class aarch64_enc_testp(iRegP src) %{
 3632     C2_MacroAssembler _masm(&cbuf);
 3633     Register reg = as_Register($src$$reg);
 3634     __ cmp(reg, zr);
 3635   %}
 3636 
 3637   enc_class aarch64_enc_testn(iRegN src) %{
 3638     C2_MacroAssembler _masm(&cbuf);
 3639     Register reg = as_Register($src$$reg);
 3640     __ cmpw(reg, zr);
 3641   %}
 3642 
 3643   enc_class aarch64_enc_b(label lbl) %{
 3644     C2_MacroAssembler _masm(&cbuf);
 3645     Label *L = $lbl$$label;
 3646     __ b(*L);
 3647   %}
 3648 
 3649   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3650     C2_MacroAssembler _masm(&cbuf);
 3651     Label *L = $lbl$$label;
 3652     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3653   %}
 3654 
 3655   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3656     C2_MacroAssembler _masm(&cbuf);
 3657     Label *L = $lbl$$label;
 3658     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3659   %}
 3660 
 3661   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3662   %{
 3663      Register sub_reg = as_Register($sub$$reg);
 3664      Register super_reg = as_Register($super$$reg);
 3665      Register temp_reg = as_Register($temp$$reg);
 3666      Register result_reg = as_Register($result$$reg);
 3667 
 3668      Label miss;
 3669      C2_MacroAssembler _masm(&cbuf);
 3670      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3671                                      NULL, &miss,
 3672                                      /*set_cond_codes:*/ true);
 3673      if ($primary) {
 3674        __ mov(result_reg, zr);
 3675      }
 3676      __ bind(miss);
 3677   %}
 3678 
 3679   enc_class aarch64_enc_java_static_call(method meth) %{
 3680     C2_MacroAssembler _masm(&cbuf);
 3681 
 3682     address addr = (address)$meth$$method;
 3683     address call;
 3684     if (!_method) {
 3685       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3686       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3687       if (call == NULL) {
 3688         ciEnv::current()->record_failure("CodeCache is full");
 3689         return;
 3690       }
 3691     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3692       // The NOP here is purely to ensure that eliding a call to
 3693       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3694       __ nop();
 3695       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3696     } else {
 3697       int method_index = resolved_method_index(cbuf);
 3698       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3699                                                   : static_call_Relocation::spec(method_index);
 3700       call = __ trampoline_call(Address(addr, rspec));
 3701       if (call == NULL) {
 3702         ciEnv::current()->record_failure("CodeCache is full");
 3703         return;
 3704       }
 3705       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3706         // Calls of the same statically bound method can share
 3707         // a stub to the interpreter.
 3708         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3709       } else {
 3710         // Emit stub for static call
 3711         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3712         if (stub == NULL) {
 3713           ciEnv::current()->record_failure("CodeCache is full");
 3714           return;
 3715         }
 3716       }
 3717     }
 3718 
 3719     __ post_call_nop();
 3720 
 3721     // Only non uncommon_trap calls need to reinitialize ptrue.
 3722     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3723       __ reinitialize_ptrue();
 3724     }
 3725   %}
 3726 
 3727   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3728     C2_MacroAssembler _masm(&cbuf);
 3729     int method_index = resolved_method_index(cbuf);
 3730     address call = __ ic_call((address)$meth$$method, method_index);
 3731     if (call == NULL) {
 3732       ciEnv::current()->record_failure("CodeCache is full");
 3733       return;
 3734     }
 3735     __ post_call_nop();
 3736     if (Compile::current()->max_vector_size() > 0) {
 3737       __ reinitialize_ptrue();
 3738     }
 3739   %}
 3740 
 3741   enc_class aarch64_enc_call_epilog() %{
 3742     C2_MacroAssembler _masm(&cbuf);
 3743     if (VerifyStackAtCalls) {
 3744       // Check that stack depth is unchanged: find majik cookie on stack
 3745       __ call_Unimplemented();
 3746     }
 3747   %}
 3748 
 3749   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3750     C2_MacroAssembler _masm(&cbuf);
 3751 
 3752     // some calls to generated routines (arraycopy code) are scheduled
 3753     // by C2 as runtime calls. if so we can call them using a br (they
 3754     // will be in a reachable segment) otherwise we have to use a blr
 3755     // which loads the absolute address into a register.
 3756     address entry = (address)$meth$$method;
 3757     CodeBlob *cb = CodeCache::find_blob(entry);
 3758     if (cb) {
 3759       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3760       if (call == NULL) {
 3761         ciEnv::current()->record_failure("CodeCache is full");
 3762         return;
 3763       }
 3764       __ post_call_nop();
 3765     } else {
 3766       Label retaddr;
 3767       __ adr(rscratch2, retaddr);
 3768       __ lea(rscratch1, RuntimeAddress(entry));
 3769       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3770       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3771       __ blr(rscratch1);
 3772       __ bind(retaddr);
 3773       __ post_call_nop();
 3774       __ add(sp, sp, 2 * wordSize);
 3775     }
 3776     if (Compile::current()->max_vector_size() > 0) {
 3777       __ reinitialize_ptrue();
 3778     }
 3779   %}
 3780 
 3781   enc_class aarch64_enc_rethrow() %{
 3782     C2_MacroAssembler _masm(&cbuf);
 3783     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3784   %}
 3785 
 3786   enc_class aarch64_enc_ret() %{
 3787     C2_MacroAssembler _masm(&cbuf);
 3788 #ifdef ASSERT
 3789     if (Compile::current()->max_vector_size() > 0) {
 3790       __ verify_ptrue();
 3791     }
 3792 #endif
 3793     __ ret(lr);
 3794   %}
 3795 
 3796   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3797     C2_MacroAssembler _masm(&cbuf);
 3798     Register target_reg = as_Register($jump_target$$reg);
 3799     __ br(target_reg);
 3800   %}
 3801 
 3802   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3803     C2_MacroAssembler _masm(&cbuf);
 3804     Register target_reg = as_Register($jump_target$$reg);
 3805     // exception oop should be in r0
 3806     // ret addr has been popped into lr
 3807     // callee expects it in r3
 3808     __ mov(r3, lr);
 3809     __ br(target_reg);
 3810   %}
 3811 
 3812   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3813     C2_MacroAssembler _masm(&cbuf);
 3814     Register oop = as_Register($object$$reg);
 3815     Register box = as_Register($box$$reg);
 3816     Register disp_hdr = as_Register($tmp$$reg);
 3817     Register tmp = as_Register($tmp2$$reg);
 3818     Label cont;
 3819     Label object_has_monitor;
 3820     Label count, no_count;
 3821 
 3822     assert_different_registers(oop, box, tmp, disp_hdr);
 3823 
 3824     // Load markWord from object into displaced_header.
 3825     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3826 
 3827     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3828       __ load_klass(tmp, oop);
 3829       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3830       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3831       __ br(Assembler::NE, cont);
 3832     }
 3833 
 3834     // Check for existing monitor
 3835     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3836 
 3837     if (LockingMode == LM_MONITOR) {
 3838       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3839       __ b(cont);
 3840     } else if (LockingMode == LM_LEGACY) {
 3841       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3842       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3843 
 3844       // Initialize the box. (Must happen before we update the object mark!)
 3845       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3846 
 3847       // Compare object markWord with an unlocked value (tmp) and if
 3848       // equal exchange the stack address of our box with object markWord.
 3849       // On failure disp_hdr contains the possibly locked markWord.
 3850       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3851                  /*release*/ true, /*weak*/ false, disp_hdr);
 3852       __ br(Assembler::EQ, cont);
 3853 
 3854       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3855 
 3856       // If the compare-and-exchange succeeded, then we found an unlocked
 3857       // object, will have now locked it will continue at label cont
 3858 
 3859       // Check if the owner is self by comparing the value in the
 3860       // markWord of object (disp_hdr) with the stack pointer.
 3861       __ mov(rscratch1, sp);
 3862       __ sub(disp_hdr, disp_hdr, rscratch1);
 3863       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3864       // If condition is true we are cont and hence we can store 0 as the
 3865       // displaced header in the box, which indicates that it is a recursive lock.
 3866       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3867       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3868       __ b(cont);
 3869     } else {
 3870       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3871       __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
 3872       __ b(count);
 3873     }
 3874 
 3875     // Handle existing monitor.
 3876     __ bind(object_has_monitor);
 3877 
 3878     // The object's monitor m is unlocked iff m->owner == NULL,
 3879     // otherwise m->owner may contain a thread or a stack address.
 3880     //
 3881     // Try to CAS m->owner from NULL to current thread.
 3882     __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
 3883     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3884                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3885 
 3886     if (LockingMode != LM_LIGHTWEIGHT) {
 3887       // Store a non-null value into the box to avoid looking like a re-entrant
 3888       // lock. The fast-path monitor unlock code checks for
 3889       // markWord::monitor_value so use markWord::unused_mark which has the
 3890       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3891       __ mov(tmp, (address)markWord::unused_mark().value());
 3892       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3893     }
 3894     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3895 
 3896     __ cmp(rscratch1, rthread);
 3897     __ br(Assembler::NE, cont); // Check for recursive locking
 3898 
 3899     // Recursive lock case
 3900     __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
 3901     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3902 
 3903     __ bind(cont);
 3904     // flag == EQ indicates success
 3905     // flag == NE indicates failure
 3906     __ br(Assembler::NE, no_count);
 3907 
 3908     __ bind(count);
 3909     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3910 
 3911     __ bind(no_count);
 3912   %}
 3913 
 3914   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3915     C2_MacroAssembler _masm(&cbuf);
 3916     Register oop = as_Register($object$$reg);
 3917     Register box = as_Register($box$$reg);
 3918     Register disp_hdr = as_Register($tmp$$reg);
 3919     Register tmp = as_Register($tmp2$$reg);
 3920     Label cont;
 3921     Label object_has_monitor;
 3922     Label count, no_count;
 3923 
 3924     assert_different_registers(oop, box, tmp, disp_hdr);
 3925 
 3926     if (LockingMode == LM_LEGACY) {
 3927       // Find the lock address and load the displaced header from the stack.
 3928       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3929 
 3930       // If the displaced header is 0, we have a recursive unlock.
 3931       __ cmp(disp_hdr, zr);
 3932       __ br(Assembler::EQ, cont);
 3933     }
 3934 
 3935     // Handle existing monitor.
 3936     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3937     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3938 
 3939     if (LockingMode == LM_MONITOR) {
 3940       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3941       __ b(cont);
 3942     } else if (LockingMode == LM_LEGACY) {
 3943       // Check if it is still a light weight lock, this is is true if we
 3944       // see the stack address of the basicLock in the markWord of the
 3945       // object.
 3946 
 3947       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3948                  /*release*/ true, /*weak*/ false, tmp);
 3949       __ b(cont);
 3950     } else {
 3951       assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 3952       __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
 3953       __ b(count);
 3954     }
 3955 
 3956     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3957 
 3958     // Handle existing monitor.
 3959     __ bind(object_has_monitor);
 3960     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3961     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3962 
 3963     if (LockingMode == LM_LIGHTWEIGHT) {
 3964       // If the owner is anonymous, we need to fix it -- in an outline stub.
 3965       Register tmp2 = disp_hdr;
 3966       __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
 3967       // We cannot use tbnz here, the target might be too far away and cannot
 3968       // be encoded.
 3969       __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
 3970       C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 3971       Compile::current()->output()->add_stub(stub);
 3972       __ br(Assembler::NE, stub->entry());
 3973       __ bind(stub->continuation());
 3974     }
 3975 
 3976     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 3977 
 3978     Label notRecursive;
 3979     __ cbz(disp_hdr, notRecursive);
 3980 
 3981     // Recursive lock
 3982     __ sub(disp_hdr, disp_hdr, 1u);
 3983     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 3984     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3985     __ b(cont);
 3986 
 3987     __ bind(notRecursive);
 3988     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
 3989     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
 3990     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3991     __ cmp(rscratch1, zr); // Sets flags for result
 3992     __ cbnz(rscratch1, cont);
 3993     // need a release store here
 3994     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
 3995     __ stlr(zr, tmp); // set unowned
 3996 
 3997     __ bind(cont);
 3998     // flag == EQ indicates success
 3999     // flag == NE indicates failure
 4000     __ br(Assembler::NE, no_count);
 4001 
 4002     __ bind(count);
 4003     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 4004 
 4005     __ bind(no_count);
 4006   %}
 4007 
 4008 %}
 4009 
 4010 //----------FRAME--------------------------------------------------------------
 4011 // Definition of frame structure and management information.
 4012 //
 4013 //  S T A C K   L A Y O U T    Allocators stack-slot number
 4014 //                             |   (to get allocators register number
 4015 //  G  Owned by    |        |  v    add OptoReg::stack0())
 4016 //  r   CALLER     |        |
 4017 //  o     |        +--------+      pad to even-align allocators stack-slot
 4018 //  w     V        |  pad0  |        numbers; owned by CALLER
 4019 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 4020 //  h     ^        |   in   |  5
 4021 //        |        |  args  |  4   Holes in incoming args owned by SELF
 4022 //  |     |        |        |  3
 4023 //  |     |        +--------+
 4024 //  V     |        | old out|      Empty on Intel, window on Sparc
 4025 //        |    old |preserve|      Must be even aligned.
 4026 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 4027 //        |        |   in   |  3   area for Intel ret address
 4028 //     Owned by    |preserve|      Empty on Sparc.
 4029 //       SELF      +--------+
 4030 //        |        |  pad2  |  2   pad to align old SP
 4031 //        |        +--------+  1
 4032 //        |        | locks  |  0
 4033 //        |        +--------+----> OptoReg::stack0(), even aligned
 4034 //        |        |  pad1  | 11   pad to align new SP
 4035 //        |        +--------+
 4036 //        |        |        | 10
 4037 //        |        | spills |  9   spills
 4038 //        V        |        |  8   (pad0 slot for callee)
 4039 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4040 //        ^        |  out   |  7
 4041 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4042 //     Owned by    +--------+
 4043 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4044 //        |    new |preserve|      Must be even-aligned.
 4045 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4046 //        |        |        |
 4047 //
 4048 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4049 //         known from SELF's arguments and the Java calling convention.
 4050 //         Region 6-7 is determined per call site.
 4051 // Note 2: If the calling convention leaves holes in the incoming argument
 4052 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4053 //         are owned by the CALLEE.  Holes should not be necessary in the
 4054 //         incoming area, as the Java calling convention is completely under
 4055 //         the control of the AD file.  Doubles can be sorted and packed to
 4056 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4057 //         varargs C calling conventions.
 4058 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4059 //         even aligned with pad0 as needed.
 4060 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4061 //           (the latter is true on Intel but is it false on AArch64?)
 4062 //         region 6-11 is even aligned; it may be padded out more so that
 4063 //         the region from SP to FP meets the minimum stack alignment.
 4064 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4065 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4066 //         SP meets the minimum alignment.
 4067 
 4068 frame %{
 4069   // These three registers define part of the calling convention
 4070   // between compiled code and the interpreter.
 4071 
 4072   // Inline Cache Register or Method for I2C.
 4073   inline_cache_reg(R12);
 4074 
 4075   // Number of stack slots consumed by locking an object
 4076   sync_stack_slots(2);
 4077 
 4078   // Compiled code's Frame Pointer
 4079   frame_pointer(R31);
 4080 
 4081   // Interpreter stores its frame pointer in a register which is
 4082   // stored to the stack by I2CAdaptors.
 4083   // I2CAdaptors convert from interpreted java to compiled java.
 4084   interpreter_frame_pointer(R29);
 4085 
 4086   // Stack alignment requirement
 4087   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4088 
 4089   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4090   // for calls to C.  Supports the var-args backing area for register parms.
 4091   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4092 
 4093   // The after-PROLOG location of the return address.  Location of
 4094   // return address specifies a type (REG or STACK) and a number
 4095   // representing the register number (i.e. - use a register name) or
 4096   // stack slot.
 4097   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4098   // Otherwise, it is above the locks and verification slot and alignment word
 4099   // TODO this may well be correct but need to check why that - 2 is there
 4100   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4101   // which folds in the space used for monitors
 4102   return_addr(STACK - 2 +
 4103               align_up((Compile::current()->in_preserve_stack_slots() +
 4104                         Compile::current()->fixed_slots()),
 4105                        stack_alignment_in_slots()));
 4106 
 4107   // Location of compiled Java return values.  Same as C for now.
 4108   return_value
 4109   %{
 4110     // TODO do we allow ideal_reg == Op_RegN???
 4111     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4112            "only return normal values");
 4113 
 4114     static const int lo[Op_RegL + 1] = { // enum name
 4115       0,                                 // Op_Node
 4116       0,                                 // Op_Set
 4117       R0_num,                            // Op_RegN
 4118       R0_num,                            // Op_RegI
 4119       R0_num,                            // Op_RegP
 4120       V0_num,                            // Op_RegF
 4121       V0_num,                            // Op_RegD
 4122       R0_num                             // Op_RegL
 4123     };
 4124 
 4125     static const int hi[Op_RegL + 1] = { // enum name
 4126       0,                                 // Op_Node
 4127       0,                                 // Op_Set
 4128       OptoReg::Bad,                      // Op_RegN
 4129       OptoReg::Bad,                      // Op_RegI
 4130       R0_H_num,                          // Op_RegP
 4131       OptoReg::Bad,                      // Op_RegF
 4132       V0_H_num,                          // Op_RegD
 4133       R0_H_num                           // Op_RegL
 4134     };
 4135 
 4136     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4137   %}
 4138 %}
 4139 
 4140 //----------ATTRIBUTES---------------------------------------------------------
 4141 //----------Operand Attributes-------------------------------------------------
 4142 op_attrib op_cost(1);        // Required cost attribute
 4143 
 4144 //----------Instruction Attributes---------------------------------------------
 4145 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4146 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4147 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4148                                 // a non-matching short branch variant
 4149                                 // of some long branch?
 4150 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4151                                 // be a power of 2) specifies the
 4152                                 // alignment that some part of the
 4153                                 // instruction (not necessarily the
 4154                                 // start) requires.  If > 1, a
 4155                                 // compute_padding() function must be
 4156                                 // provided for the instruction
 4157 
 4158 //----------OPERANDS-----------------------------------------------------------
 4159 // Operand definitions must precede instruction definitions for correct parsing
 4160 // in the ADLC because operands constitute user defined types which are used in
 4161 // instruction definitions.
 4162 
 4163 //----------Simple Operands----------------------------------------------------
 4164 
 4165 // Integer operands 32 bit
 4166 // 32 bit immediate
 4167 operand immI()
 4168 %{
 4169   match(ConI);
 4170 
 4171   op_cost(0);
 4172   format %{ %}
 4173   interface(CONST_INTER);
 4174 %}
 4175 
 4176 // 32 bit zero
 4177 operand immI0()
 4178 %{
 4179   predicate(n->get_int() == 0);
 4180   match(ConI);
 4181 
 4182   op_cost(0);
 4183   format %{ %}
 4184   interface(CONST_INTER);
 4185 %}
 4186 
 4187 // 32 bit unit increment
 4188 operand immI_1()
 4189 %{
 4190   predicate(n->get_int() == 1);
 4191   match(ConI);
 4192 
 4193   op_cost(0);
 4194   format %{ %}
 4195   interface(CONST_INTER);
 4196 %}
 4197 
 4198 // 32 bit unit decrement
 4199 operand immI_M1()
 4200 %{
 4201   predicate(n->get_int() == -1);
 4202   match(ConI);
 4203 
 4204   op_cost(0);
 4205   format %{ %}
 4206   interface(CONST_INTER);
 4207 %}
 4208 
 4209 // Shift values for add/sub extension shift
 4210 operand immIExt()
 4211 %{
 4212   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immI_gt_1()
 4221 %{
 4222   predicate(n->get_int() > 1);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 operand immI_le_4()
 4231 %{
 4232   predicate(n->get_int() <= 4);
 4233   match(ConI);
 4234 
 4235   op_cost(0);
 4236   format %{ %}
 4237   interface(CONST_INTER);
 4238 %}
 4239 
 4240 operand immI_16()
 4241 %{
 4242   predicate(n->get_int() == 16);
 4243   match(ConI);
 4244 
 4245   op_cost(0);
 4246   format %{ %}
 4247   interface(CONST_INTER);
 4248 %}
 4249 
 4250 operand immI_24()
 4251 %{
 4252   predicate(n->get_int() == 24);
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 operand immI_32()
 4261 %{
 4262   predicate(n->get_int() == 32);
 4263   match(ConI);
 4264 
 4265   op_cost(0);
 4266   format %{ %}
 4267   interface(CONST_INTER);
 4268 %}
 4269 
 4270 operand immI_48()
 4271 %{
 4272   predicate(n->get_int() == 48);
 4273   match(ConI);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 operand immI_56()
 4281 %{
 4282   predicate(n->get_int() == 56);
 4283   match(ConI);
 4284 
 4285   op_cost(0);
 4286   format %{ %}
 4287   interface(CONST_INTER);
 4288 %}
 4289 
 4290 operand immI_63()
 4291 %{
 4292   predicate(n->get_int() == 63);
 4293   match(ConI);
 4294 
 4295   op_cost(0);
 4296   format %{ %}
 4297   interface(CONST_INTER);
 4298 %}
 4299 
 4300 operand immI_64()
 4301 %{
 4302   predicate(n->get_int() == 64);
 4303   match(ConI);
 4304 
 4305   op_cost(0);
 4306   format %{ %}
 4307   interface(CONST_INTER);
 4308 %}
 4309 
 4310 operand immI_255()
 4311 %{
 4312   predicate(n->get_int() == 255);
 4313   match(ConI);
 4314 
 4315   op_cost(0);
 4316   format %{ %}
 4317   interface(CONST_INTER);
 4318 %}
 4319 
 4320 operand immI_65535()
 4321 %{
 4322   predicate(n->get_int() == 65535);
 4323   match(ConI);
 4324 
 4325   op_cost(0);
 4326   format %{ %}
 4327   interface(CONST_INTER);
 4328 %}
 4329 
 4330 operand immI_positive()
 4331 %{
 4332   predicate(n->get_int() > 0);
 4333   match(ConI);
 4334 
 4335   op_cost(0);
 4336   format %{ %}
 4337   interface(CONST_INTER);
 4338 %}
 4339 
 4340 // BoolTest condition for signed compare
 4341 operand immI_cmp_cond()
 4342 %{
 4343   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4344   match(ConI);
 4345 
 4346   op_cost(0);
 4347   format %{ %}
 4348   interface(CONST_INTER);
 4349 %}
 4350 
 4351 // BoolTest condition for unsigned compare
 4352 operand immI_cmpU_cond()
 4353 %{
 4354   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4355   match(ConI);
 4356 
 4357   op_cost(0);
 4358   format %{ %}
 4359   interface(CONST_INTER);
 4360 %}
 4361 
 4362 operand immL_255()
 4363 %{
 4364   predicate(n->get_long() == 255L);
 4365   match(ConL);
 4366 
 4367   op_cost(0);
 4368   format %{ %}
 4369   interface(CONST_INTER);
 4370 %}
 4371 
 4372 operand immL_65535()
 4373 %{
 4374   predicate(n->get_long() == 65535L);
 4375   match(ConL);
 4376 
 4377   op_cost(0);
 4378   format %{ %}
 4379   interface(CONST_INTER);
 4380 %}
 4381 
 4382 operand immL_4294967295()
 4383 %{
 4384   predicate(n->get_long() == 4294967295L);
 4385   match(ConL);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 operand immL_bitmask()
 4393 %{
 4394   predicate((n->get_long() != 0)
 4395             && ((n->get_long() & 0xc000000000000000l) == 0)
 4396             && is_power_of_2(n->get_long() + 1));
 4397   match(ConL);
 4398 
 4399   op_cost(0);
 4400   format %{ %}
 4401   interface(CONST_INTER);
 4402 %}
 4403 
 4404 operand immI_bitmask()
 4405 %{
 4406   predicate((n->get_int() != 0)
 4407             && ((n->get_int() & 0xc0000000) == 0)
 4408             && is_power_of_2(n->get_int() + 1));
 4409   match(ConI);
 4410 
 4411   op_cost(0);
 4412   format %{ %}
 4413   interface(CONST_INTER);
 4414 %}
 4415 
 4416 operand immL_positive_bitmaskI()
 4417 %{
 4418   predicate((n->get_long() != 0)
 4419             && ((julong)n->get_long() < 0x80000000ULL)
 4420             && is_power_of_2(n->get_long() + 1));
 4421   match(ConL);
 4422 
 4423   op_cost(0);
 4424   format %{ %}
 4425   interface(CONST_INTER);
 4426 %}
 4427 
 4428 // Scale values for scaled offset addressing modes (up to long but not quad)
 4429 operand immIScale()
 4430 %{
 4431   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4432   match(ConI);
 4433 
 4434   op_cost(0);
 4435   format %{ %}
 4436   interface(CONST_INTER);
 4437 %}
 4438 
 4439 // 26 bit signed offset -- for pc-relative branches
 4440 operand immI26()
 4441 %{
 4442   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4443   match(ConI);
 4444 
 4445   op_cost(0);
 4446   format %{ %}
 4447   interface(CONST_INTER);
 4448 %}
 4449 
 4450 // 19 bit signed offset -- for pc-relative loads
 4451 operand immI19()
 4452 %{
 4453   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4454   match(ConI);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 // 5 bit signed integer
 4462 operand immI5()
 4463 %{
 4464   predicate(Assembler::is_simm(n->get_int(), 5));
 4465   match(ConI);
 4466 
 4467   op_cost(0);
 4468   format %{ %}
 4469   interface(CONST_INTER);
 4470 %}
 4471 
 4472 // 7 bit unsigned integer
 4473 operand immIU7()
 4474 %{
 4475   predicate(Assembler::is_uimm(n->get_int(), 7));
 4476   match(ConI);
 4477 
 4478   op_cost(0);
 4479   format %{ %}
 4480   interface(CONST_INTER);
 4481 %}
 4482 
 4483 // 12 bit unsigned offset -- for base plus immediate loads
 4484 operand immIU12()
 4485 %{
 4486   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4487   match(ConI);
 4488 
 4489   op_cost(0);
 4490   format %{ %}
 4491   interface(CONST_INTER);
 4492 %}
 4493 
 4494 operand immLU12()
 4495 %{
 4496   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4497   match(ConL);
 4498 
 4499   op_cost(0);
 4500   format %{ %}
 4501   interface(CONST_INTER);
 4502 %}
 4503 
 4504 // Offset for scaled or unscaled immediate loads and stores
 4505 operand immIOffset()
 4506 %{
 4507   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4508   match(ConI);
 4509 
 4510   op_cost(0);
 4511   format %{ %}
 4512   interface(CONST_INTER);
 4513 %}
 4514 
 4515 operand immIOffset1()
 4516 %{
 4517   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4518   match(ConI);
 4519 
 4520   op_cost(0);
 4521   format %{ %}
 4522   interface(CONST_INTER);
 4523 %}
 4524 
 4525 operand immIOffset2()
 4526 %{
 4527   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4528   match(ConI);
 4529 
 4530   op_cost(0);
 4531   format %{ %}
 4532   interface(CONST_INTER);
 4533 %}
 4534 
 4535 operand immIOffset4()
 4536 %{
 4537   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4538   match(ConI);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 operand immIOffset8()
 4546 %{
 4547   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4548   match(ConI);
 4549 
 4550   op_cost(0);
 4551   format %{ %}
 4552   interface(CONST_INTER);
 4553 %}
 4554 
 4555 operand immIOffset16()
 4556 %{
 4557   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4558   match(ConI);
 4559 
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 operand immLoffset()
 4566 %{
 4567   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4568   match(ConL);
 4569 
 4570   op_cost(0);
 4571   format %{ %}
 4572   interface(CONST_INTER);
 4573 %}
 4574 
 4575 operand immLoffset1()
 4576 %{
 4577   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4578   match(ConL);
 4579 
 4580   op_cost(0);
 4581   format %{ %}
 4582   interface(CONST_INTER);
 4583 %}
 4584 
 4585 operand immLoffset2()
 4586 %{
 4587   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4588   match(ConL);
 4589 
 4590   op_cost(0);
 4591   format %{ %}
 4592   interface(CONST_INTER);
 4593 %}
 4594 
 4595 operand immLoffset4()
 4596 %{
 4597   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4598   match(ConL);
 4599 
 4600   op_cost(0);
 4601   format %{ %}
 4602   interface(CONST_INTER);
 4603 %}
 4604 
 4605 operand immLoffset8()
 4606 %{
 4607   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4608   match(ConL);
 4609 
 4610   op_cost(0);
 4611   format %{ %}
 4612   interface(CONST_INTER);
 4613 %}
 4614 
 4615 operand immLoffset16()
 4616 %{
 4617   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4618   match(ConL);
 4619 
 4620   op_cost(0);
 4621   format %{ %}
 4622   interface(CONST_INTER);
 4623 %}
 4624 
 4625 // 5 bit signed long integer
 4626 operand immL5()
 4627 %{
 4628   predicate(Assembler::is_simm(n->get_long(), 5));
 4629   match(ConL);
 4630 
 4631   op_cost(0);
 4632   format %{ %}
 4633   interface(CONST_INTER);
 4634 %}
 4635 
 4636 // 7 bit unsigned long integer
 4637 operand immLU7()
 4638 %{
 4639   predicate(Assembler::is_uimm(n->get_long(), 7));
 4640   match(ConL);
 4641 
 4642   op_cost(0);
 4643   format %{ %}
 4644   interface(CONST_INTER);
 4645 %}
 4646 
 4647 // 8 bit signed value.
 4648 operand immI8()
 4649 %{
 4650   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4651   match(ConI);
 4652 
 4653   op_cost(0);
 4654   format %{ %}
 4655   interface(CONST_INTER);
 4656 %}
 4657 
 4658 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4659 operand immI8_shift8()
 4660 %{
 4661   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4662             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4663   match(ConI);
 4664 
 4665   op_cost(0);
 4666   format %{ %}
 4667   interface(CONST_INTER);
 4668 %}
 4669 
 4670 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4671 operand immL8_shift8()
 4672 %{
 4673   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4674             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4675   match(ConL);
 4676 
 4677   op_cost(0);
 4678   format %{ %}
 4679   interface(CONST_INTER);
 4680 %}
 4681 
 4682 // 8 bit integer valid for vector add sub immediate
 4683 operand immBAddSubV()
 4684 %{
 4685   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4686   match(ConI);
 4687 
 4688   op_cost(0);
 4689   format %{ %}
 4690   interface(CONST_INTER);
 4691 %}
 4692 
 4693 // 32 bit integer valid for add sub immediate
 4694 operand immIAddSub()
 4695 %{
 4696   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4697   match(ConI);
 4698   op_cost(0);
 4699   format %{ %}
 4700   interface(CONST_INTER);
 4701 %}
 4702 
 4703 // 32 bit integer valid for vector add sub immediate
 4704 operand immIAddSubV()
 4705 %{
 4706   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4707   match(ConI);
 4708 
 4709   op_cost(0);
 4710   format %{ %}
 4711   interface(CONST_INTER);
 4712 %}
 4713 
 4714 // 32 bit unsigned integer valid for logical immediate
 4715 
 4716 operand immBLog()
 4717 %{
 4718   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4719   match(ConI);
 4720 
 4721   op_cost(0);
 4722   format %{ %}
 4723   interface(CONST_INTER);
 4724 %}
 4725 
 4726 operand immSLog()
 4727 %{
 4728   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4729   match(ConI);
 4730 
 4731   op_cost(0);
 4732   format %{ %}
 4733   interface(CONST_INTER);
 4734 %}
 4735 
 4736 operand immILog()
 4737 %{
 4738   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4739   match(ConI);
 4740 
 4741   op_cost(0);
 4742   format %{ %}
 4743   interface(CONST_INTER);
 4744 %}
 4745 
 4746 // Integer operands 64 bit
 4747 // 64 bit immediate
 4748 operand immL()
 4749 %{
 4750   match(ConL);
 4751 
 4752   op_cost(0);
 4753   format %{ %}
 4754   interface(CONST_INTER);
 4755 %}
 4756 
 4757 // 64 bit zero
 4758 operand immL0()
 4759 %{
 4760   predicate(n->get_long() == 0);
 4761   match(ConL);
 4762 
 4763   op_cost(0);
 4764   format %{ %}
 4765   interface(CONST_INTER);
 4766 %}
 4767 
 4768 // 64 bit unit increment
 4769 operand immL_1()
 4770 %{
 4771   predicate(n->get_long() == 1);
 4772   match(ConL);
 4773 
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(CONST_INTER);
 4777 %}
 4778 
 4779 // 64 bit unit decrement
 4780 operand immL_M1()
 4781 %{
 4782   predicate(n->get_long() == -1);
 4783   match(ConL);
 4784 
 4785   op_cost(0);
 4786   format %{ %}
 4787   interface(CONST_INTER);
 4788 %}
 4789 
 4790 // 32 bit offset of pc in thread anchor
 4791 
 4792 operand immL_pc_off()
 4793 %{
 4794   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4795                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4796   match(ConL);
 4797 
 4798   op_cost(0);
 4799   format %{ %}
 4800   interface(CONST_INTER);
 4801 %}
 4802 
 4803 // 64 bit integer valid for add sub immediate
 4804 operand immLAddSub()
 4805 %{
 4806   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4807   match(ConL);
 4808   op_cost(0);
 4809   format %{ %}
 4810   interface(CONST_INTER);
 4811 %}
 4812 
 4813 // 64 bit integer valid for addv subv immediate
 4814 operand immLAddSubV()
 4815 %{
 4816   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4817   match(ConL);
 4818 
 4819   op_cost(0);
 4820   format %{ %}
 4821   interface(CONST_INTER);
 4822 %}
 4823 
 4824 // 64 bit integer valid for logical immediate
 4825 operand immLLog()
 4826 %{
 4827   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4828   match(ConL);
 4829   op_cost(0);
 4830   format %{ %}
 4831   interface(CONST_INTER);
 4832 %}
 4833 
 4834 // Long Immediate: low 32-bit mask
 4835 operand immL_32bits()
 4836 %{
 4837   predicate(n->get_long() == 0xFFFFFFFFL);
 4838   match(ConL);
 4839   op_cost(0);
 4840   format %{ %}
 4841   interface(CONST_INTER);
 4842 %}
 4843 
 4844 // Pointer operands
 4845 // Pointer Immediate
 4846 operand immP()
 4847 %{
 4848   match(ConP);
 4849 
 4850   op_cost(0);
 4851   format %{ %}
 4852   interface(CONST_INTER);
 4853 %}
 4854 
 4855 // NULL Pointer Immediate
 4856 operand immP0()
 4857 %{
 4858   predicate(n->get_ptr() == 0);
 4859   match(ConP);
 4860 
 4861   op_cost(0);
 4862   format %{ %}
 4863   interface(CONST_INTER);
 4864 %}
 4865 
 4866 // Pointer Immediate One
 4867 // this is used in object initialization (initial object header)
 4868 operand immP_1()
 4869 %{
 4870   predicate(n->get_ptr() == 1);
 4871   match(ConP);
 4872 
 4873   op_cost(0);
 4874   format %{ %}
 4875   interface(CONST_INTER);
 4876 %}
 4877 
 4878 // Card Table Byte Map Base
 4879 operand immByteMapBase()
 4880 %{
 4881   // Get base of card map
 4882   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4883             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4884   match(ConP);
 4885 
 4886   op_cost(0);
 4887   format %{ %}
 4888   interface(CONST_INTER);
 4889 %}
 4890 
 4891 // Pointer Immediate Minus One
 4892 // this is used when we want to write the current PC to the thread anchor
 4893 operand immP_M1()
 4894 %{
 4895   predicate(n->get_ptr() == -1);
 4896   match(ConP);
 4897 
 4898   op_cost(0);
 4899   format %{ %}
 4900   interface(CONST_INTER);
 4901 %}
 4902 
 4903 // Pointer Immediate Minus Two
 4904 // this is used when we want to write the current PC to the thread anchor
 4905 operand immP_M2()
 4906 %{
 4907   predicate(n->get_ptr() == -2);
 4908   match(ConP);
 4909 
 4910   op_cost(0);
 4911   format %{ %}
 4912   interface(CONST_INTER);
 4913 %}
 4914 
 4915 // Float and Double operands
 4916 // Double Immediate
 4917 operand immD()
 4918 %{
 4919   match(ConD);
 4920   op_cost(0);
 4921   format %{ %}
 4922   interface(CONST_INTER);
 4923 %}
 4924 
 4925 // Double Immediate: +0.0d
 4926 operand immD0()
 4927 %{
 4928   predicate(jlong_cast(n->getd()) == 0);
 4929   match(ConD);
 4930 
 4931   op_cost(0);
 4932   format %{ %}
 4933   interface(CONST_INTER);
 4934 %}
 4935 
 4936 // constant 'double +0.0'.
 4937 operand immDPacked()
 4938 %{
 4939   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4940   match(ConD);
 4941   op_cost(0);
 4942   format %{ %}
 4943   interface(CONST_INTER);
 4944 %}
 4945 
 4946 // Float Immediate
 4947 operand immF()
 4948 %{
 4949   match(ConF);
 4950   op_cost(0);
 4951   format %{ %}
 4952   interface(CONST_INTER);
 4953 %}
 4954 
 4955 // Float Immediate: +0.0f.
 4956 operand immF0()
 4957 %{
 4958   predicate(jint_cast(n->getf()) == 0);
 4959   match(ConF);
 4960 
 4961   op_cost(0);
 4962   format %{ %}
 4963   interface(CONST_INTER);
 4964 %}
 4965 
 4966 //
 4967 operand immFPacked()
 4968 %{
 4969   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4970   match(ConF);
 4971   op_cost(0);
 4972   format %{ %}
 4973   interface(CONST_INTER);
 4974 %}
 4975 
 4976 // Narrow pointer operands
 4977 // Narrow Pointer Immediate
 4978 operand immN()
 4979 %{
 4980   match(ConN);
 4981 
 4982   op_cost(0);
 4983   format %{ %}
 4984   interface(CONST_INTER);
 4985 %}
 4986 
 4987 // Narrow NULL Pointer Immediate
 4988 operand immN0()
 4989 %{
 4990   predicate(n->get_narrowcon() == 0);
 4991   match(ConN);
 4992 
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(CONST_INTER);
 4996 %}
 4997 
 4998 operand immNKlass()
 4999 %{
 5000   match(ConNKlass);
 5001 
 5002   op_cost(0);
 5003   format %{ %}
 5004   interface(CONST_INTER);
 5005 %}
 5006 
 5007 // Integer 32 bit Register Operands
 5008 // Integer 32 bitRegister (excludes SP)
 5009 operand iRegI()
 5010 %{
 5011   constraint(ALLOC_IN_RC(any_reg32));
 5012   match(RegI);
 5013   match(iRegINoSp);
 5014   op_cost(0);
 5015   format %{ %}
 5016   interface(REG_INTER);
 5017 %}
 5018 
 5019 // Integer 32 bit Register not Special
 5020 operand iRegINoSp()
 5021 %{
 5022   constraint(ALLOC_IN_RC(no_special_reg32));
 5023   match(RegI);
 5024   op_cost(0);
 5025   format %{ %}
 5026   interface(REG_INTER);
 5027 %}
 5028 
 5029 // Integer 64 bit Register Operands
 5030 // Integer 64 bit Register (includes SP)
 5031 operand iRegL()
 5032 %{
 5033   constraint(ALLOC_IN_RC(any_reg));
 5034   match(RegL);
 5035   match(iRegLNoSp);
 5036   op_cost(0);
 5037   format %{ %}
 5038   interface(REG_INTER);
 5039 %}
 5040 
 5041 // Integer 64 bit Register not Special
 5042 operand iRegLNoSp()
 5043 %{
 5044   constraint(ALLOC_IN_RC(no_special_reg));
 5045   match(RegL);
 5046   match(iRegL_R0);
 5047   format %{ %}
 5048   interface(REG_INTER);
 5049 %}
 5050 
 5051 // Pointer Register Operands
 5052 // Pointer Register
 5053 operand iRegP()
 5054 %{
 5055   constraint(ALLOC_IN_RC(ptr_reg));
 5056   match(RegP);
 5057   match(iRegPNoSp);
 5058   match(iRegP_R0);
 5059   //match(iRegP_R2);
 5060   //match(iRegP_R4);
 5061   match(iRegP_R5);
 5062   match(thread_RegP);
 5063   op_cost(0);
 5064   format %{ %}
 5065   interface(REG_INTER);
 5066 %}
 5067 
 5068 // Pointer 64 bit Register not Special
 5069 operand iRegPNoSp()
 5070 %{
 5071   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 5072   match(RegP);
 5073   // match(iRegP);
 5074   // match(iRegP_R0);
 5075   // match(iRegP_R2);
 5076   // match(iRegP_R4);
 5077   // match(iRegP_R5);
 5078   // match(thread_RegP);
 5079   op_cost(0);
 5080   format %{ %}
 5081   interface(REG_INTER);
 5082 %}
 5083 
 5084 // Pointer 64 bit Register R0 only
 5085 operand iRegP_R0()
 5086 %{
 5087   constraint(ALLOC_IN_RC(r0_reg));
 5088   match(RegP);
 5089   // match(iRegP);
 5090   match(iRegPNoSp);
 5091   op_cost(0);
 5092   format %{ %}
 5093   interface(REG_INTER);
 5094 %}
 5095 
 5096 // Pointer 64 bit Register R1 only
 5097 operand iRegP_R1()
 5098 %{
 5099   constraint(ALLOC_IN_RC(r1_reg));
 5100   match(RegP);
 5101   // match(iRegP);
 5102   match(iRegPNoSp);
 5103   op_cost(0);
 5104   format %{ %}
 5105   interface(REG_INTER);
 5106 %}
 5107 
 5108 // Pointer 64 bit Register R2 only
 5109 operand iRegP_R2()
 5110 %{
 5111   constraint(ALLOC_IN_RC(r2_reg));
 5112   match(RegP);
 5113   // match(iRegP);
 5114   match(iRegPNoSp);
 5115   op_cost(0);
 5116   format %{ %}
 5117   interface(REG_INTER);
 5118 %}
 5119 
 5120 // Pointer 64 bit Register R3 only
 5121 operand iRegP_R3()
 5122 %{
 5123   constraint(ALLOC_IN_RC(r3_reg));
 5124   match(RegP);
 5125   // match(iRegP);
 5126   match(iRegPNoSp);
 5127   op_cost(0);
 5128   format %{ %}
 5129   interface(REG_INTER);
 5130 %}
 5131 
 5132 // Pointer 64 bit Register R4 only
 5133 operand iRegP_R4()
 5134 %{
 5135   constraint(ALLOC_IN_RC(r4_reg));
 5136   match(RegP);
 5137   // match(iRegP);
 5138   match(iRegPNoSp);
 5139   op_cost(0);
 5140   format %{ %}
 5141   interface(REG_INTER);
 5142 %}
 5143 
 5144 // Pointer 64 bit Register R5 only
 5145 operand iRegP_R5()
 5146 %{
 5147   constraint(ALLOC_IN_RC(r5_reg));
 5148   match(RegP);
 5149   // match(iRegP);
 5150   match(iRegPNoSp);
 5151   op_cost(0);
 5152   format %{ %}
 5153   interface(REG_INTER);
 5154 %}
 5155 
 5156 // Pointer 64 bit Register R10 only
 5157 operand iRegP_R10()
 5158 %{
 5159   constraint(ALLOC_IN_RC(r10_reg));
 5160   match(RegP);
 5161   // match(iRegP);
 5162   match(iRegPNoSp);
 5163   op_cost(0);
 5164   format %{ %}
 5165   interface(REG_INTER);
 5166 %}
 5167 
 5168 // Long 64 bit Register R0 only
 5169 operand iRegL_R0()
 5170 %{
 5171   constraint(ALLOC_IN_RC(r0_reg));
 5172   match(RegL);
 5173   match(iRegLNoSp);
 5174   op_cost(0);
 5175   format %{ %}
 5176   interface(REG_INTER);
 5177 %}
 5178 
 5179 // Long 64 bit Register R2 only
 5180 operand iRegL_R2()
 5181 %{
 5182   constraint(ALLOC_IN_RC(r2_reg));
 5183   match(RegL);
 5184   match(iRegLNoSp);
 5185   op_cost(0);
 5186   format %{ %}
 5187   interface(REG_INTER);
 5188 %}
 5189 
 5190 // Long 64 bit Register R3 only
 5191 operand iRegL_R3()
 5192 %{
 5193   constraint(ALLOC_IN_RC(r3_reg));
 5194   match(RegL);
 5195   match(iRegLNoSp);
 5196   op_cost(0);
 5197   format %{ %}
 5198   interface(REG_INTER);
 5199 %}
 5200 
 5201 // Long 64 bit Register R11 only
 5202 operand iRegL_R11()
 5203 %{
 5204   constraint(ALLOC_IN_RC(r11_reg));
 5205   match(RegL);
 5206   match(iRegLNoSp);
 5207   op_cost(0);
 5208   format %{ %}
 5209   interface(REG_INTER);
 5210 %}
 5211 
 5212 // Pointer 64 bit Register FP only
 5213 operand iRegP_FP()
 5214 %{
 5215   constraint(ALLOC_IN_RC(fp_reg));
 5216   match(RegP);
 5217   // match(iRegP);
 5218   op_cost(0);
 5219   format %{ %}
 5220   interface(REG_INTER);
 5221 %}
 5222 
 5223 // Register R0 only
 5224 operand iRegI_R0()
 5225 %{
 5226   constraint(ALLOC_IN_RC(int_r0_reg));
 5227   match(RegI);
 5228   match(iRegINoSp);
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 // Register R2 only
 5235 operand iRegI_R2()
 5236 %{
 5237   constraint(ALLOC_IN_RC(int_r2_reg));
 5238   match(RegI);
 5239   match(iRegINoSp);
 5240   op_cost(0);
 5241   format %{ %}
 5242   interface(REG_INTER);
 5243 %}
 5244 
 5245 // Register R3 only
 5246 operand iRegI_R3()
 5247 %{
 5248   constraint(ALLOC_IN_RC(int_r3_reg));
 5249   match(RegI);
 5250   match(iRegINoSp);
 5251   op_cost(0);
 5252   format %{ %}
 5253   interface(REG_INTER);
 5254 %}
 5255 
 5256 
 5257 // Register R4 only
 5258 operand iRegI_R4()
 5259 %{
 5260   constraint(ALLOC_IN_RC(int_r4_reg));
 5261   match(RegI);
 5262   match(iRegINoSp);
 5263   op_cost(0);
 5264   format %{ %}
 5265   interface(REG_INTER);
 5266 %}
 5267 
 5268 
 5269 // Pointer Register Operands
 5270 // Narrow Pointer Register
 5271 operand iRegN()
 5272 %{
 5273   constraint(ALLOC_IN_RC(any_reg32));
 5274   match(RegN);
 5275   match(iRegNNoSp);
 5276   op_cost(0);
 5277   format %{ %}
 5278   interface(REG_INTER);
 5279 %}
 5280 
 5281 operand iRegN_R0()
 5282 %{
 5283   constraint(ALLOC_IN_RC(r0_reg));
 5284   match(iRegN);
 5285   op_cost(0);
 5286   format %{ %}
 5287   interface(REG_INTER);
 5288 %}
 5289 
 5290 operand iRegN_R2()
 5291 %{
 5292   constraint(ALLOC_IN_RC(r2_reg));
 5293   match(iRegN);
 5294   op_cost(0);
 5295   format %{ %}
 5296   interface(REG_INTER);
 5297 %}
 5298 
 5299 operand iRegN_R3()
 5300 %{
 5301   constraint(ALLOC_IN_RC(r3_reg));
 5302   match(iRegN);
 5303   op_cost(0);
 5304   format %{ %}
 5305   interface(REG_INTER);
 5306 %}
 5307 
 5308 // Integer 64 bit Register not Special
 5309 operand iRegNNoSp()
 5310 %{
 5311   constraint(ALLOC_IN_RC(no_special_reg32));
 5312   match(RegN);
 5313   op_cost(0);
 5314   format %{ %}
 5315   interface(REG_INTER);
 5316 %}
 5317 
 5318 // Float Register
 5319 // Float register operands
 5320 operand vRegF()
 5321 %{
 5322   constraint(ALLOC_IN_RC(float_reg));
 5323   match(RegF);
 5324 
 5325   op_cost(0);
 5326   format %{ %}
 5327   interface(REG_INTER);
 5328 %}
 5329 
 5330 // Double Register
 5331 // Double register operands
 5332 operand vRegD()
 5333 %{
 5334   constraint(ALLOC_IN_RC(double_reg));
 5335   match(RegD);
 5336 
 5337   op_cost(0);
 5338   format %{ %}
 5339   interface(REG_INTER);
 5340 %}
 5341 
 5342 // Generic vector class. This will be used for
 5343 // all vector operands, including NEON and SVE.
 5344 operand vReg()
 5345 %{
 5346   constraint(ALLOC_IN_RC(dynamic));
 5347   match(VecA);
 5348   match(VecD);
 5349   match(VecX);
 5350 
 5351   op_cost(0);
 5352   format %{ %}
 5353   interface(REG_INTER);
 5354 %}
 5355 
 5356 operand vecA()
 5357 %{
 5358   constraint(ALLOC_IN_RC(vectora_reg));
 5359   match(VecA);
 5360 
 5361   op_cost(0);
 5362   format %{ %}
 5363   interface(REG_INTER);
 5364 %}
 5365 
 5366 operand vecD()
 5367 %{
 5368   constraint(ALLOC_IN_RC(vectord_reg));
 5369   match(VecD);
 5370 
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vecX()
 5377 %{
 5378   constraint(ALLOC_IN_RC(vectorx_reg));
 5379   match(VecX);
 5380 
 5381   op_cost(0);
 5382   format %{ %}
 5383   interface(REG_INTER);
 5384 %}
 5385 
 5386 operand vRegD_V0()
 5387 %{
 5388   constraint(ALLOC_IN_RC(v0_reg));
 5389   match(RegD);
 5390   op_cost(0);
 5391   format %{ %}
 5392   interface(REG_INTER);
 5393 %}
 5394 
 5395 operand vRegD_V1()
 5396 %{
 5397   constraint(ALLOC_IN_RC(v1_reg));
 5398   match(RegD);
 5399   op_cost(0);
 5400   format %{ %}
 5401   interface(REG_INTER);
 5402 %}
 5403 
 5404 operand vRegD_V2()
 5405 %{
 5406   constraint(ALLOC_IN_RC(v2_reg));
 5407   match(RegD);
 5408   op_cost(0);
 5409   format %{ %}
 5410   interface(REG_INTER);
 5411 %}
 5412 
 5413 operand vRegD_V3()
 5414 %{
 5415   constraint(ALLOC_IN_RC(v3_reg));
 5416   match(RegD);
 5417   op_cost(0);
 5418   format %{ %}
 5419   interface(REG_INTER);
 5420 %}
 5421 
 5422 operand vRegD_V4()
 5423 %{
 5424   constraint(ALLOC_IN_RC(v4_reg));
 5425   match(RegD);
 5426   op_cost(0);
 5427   format %{ %}
 5428   interface(REG_INTER);
 5429 %}
 5430 
 5431 operand vRegD_V5()
 5432 %{
 5433   constraint(ALLOC_IN_RC(v5_reg));
 5434   match(RegD);
 5435   op_cost(0);
 5436   format %{ %}
 5437   interface(REG_INTER);
 5438 %}
 5439 
 5440 operand vRegD_V6()
 5441 %{
 5442   constraint(ALLOC_IN_RC(v6_reg));
 5443   match(RegD);
 5444   op_cost(0);
 5445   format %{ %}
 5446   interface(REG_INTER);
 5447 %}
 5448 
 5449 operand vRegD_V7()
 5450 %{
 5451   constraint(ALLOC_IN_RC(v7_reg));
 5452   match(RegD);
 5453   op_cost(0);
 5454   format %{ %}
 5455   interface(REG_INTER);
 5456 %}
 5457 
 5458 operand vRegD_V8()
 5459 %{
 5460   constraint(ALLOC_IN_RC(v8_reg));
 5461   match(RegD);
 5462   op_cost(0);
 5463   format %{ %}
 5464   interface(REG_INTER);
 5465 %}
 5466 
 5467 operand vRegD_V9()
 5468 %{
 5469   constraint(ALLOC_IN_RC(v9_reg));
 5470   match(RegD);
 5471   op_cost(0);
 5472   format %{ %}
 5473   interface(REG_INTER);
 5474 %}
 5475 
 5476 operand vRegD_V10()
 5477 %{
 5478   constraint(ALLOC_IN_RC(v10_reg));
 5479   match(RegD);
 5480   op_cost(0);
 5481   format %{ %}
 5482   interface(REG_INTER);
 5483 %}
 5484 
 5485 operand vRegD_V11()
 5486 %{
 5487   constraint(ALLOC_IN_RC(v11_reg));
 5488   match(RegD);
 5489   op_cost(0);
 5490   format %{ %}
 5491   interface(REG_INTER);
 5492 %}
 5493 
 5494 operand vRegD_V12()
 5495 %{
 5496   constraint(ALLOC_IN_RC(v12_reg));
 5497   match(RegD);
 5498   op_cost(0);
 5499   format %{ %}
 5500   interface(REG_INTER);
 5501 %}
 5502 
 5503 operand vRegD_V13()
 5504 %{
 5505   constraint(ALLOC_IN_RC(v13_reg));
 5506   match(RegD);
 5507   op_cost(0);
 5508   format %{ %}
 5509   interface(REG_INTER);
 5510 %}
 5511 
 5512 operand vRegD_V14()
 5513 %{
 5514   constraint(ALLOC_IN_RC(v14_reg));
 5515   match(RegD);
 5516   op_cost(0);
 5517   format %{ %}
 5518   interface(REG_INTER);
 5519 %}
 5520 
 5521 operand vRegD_V15()
 5522 %{
 5523   constraint(ALLOC_IN_RC(v15_reg));
 5524   match(RegD);
 5525   op_cost(0);
 5526   format %{ %}
 5527   interface(REG_INTER);
 5528 %}
 5529 
 5530 operand vRegD_V16()
 5531 %{
 5532   constraint(ALLOC_IN_RC(v16_reg));
 5533   match(RegD);
 5534   op_cost(0);
 5535   format %{ %}
 5536   interface(REG_INTER);
 5537 %}
 5538 
 5539 operand vRegD_V17()
 5540 %{
 5541   constraint(ALLOC_IN_RC(v17_reg));
 5542   match(RegD);
 5543   op_cost(0);
 5544   format %{ %}
 5545   interface(REG_INTER);
 5546 %}
 5547 
 5548 operand vRegD_V18()
 5549 %{
 5550   constraint(ALLOC_IN_RC(v18_reg));
 5551   match(RegD);
 5552   op_cost(0);
 5553   format %{ %}
 5554   interface(REG_INTER);
 5555 %}
 5556 
 5557 operand vRegD_V19()
 5558 %{
 5559   constraint(ALLOC_IN_RC(v19_reg));
 5560   match(RegD);
 5561   op_cost(0);
 5562   format %{ %}
 5563   interface(REG_INTER);
 5564 %}
 5565 
 5566 operand vRegD_V20()
 5567 %{
 5568   constraint(ALLOC_IN_RC(v20_reg));
 5569   match(RegD);
 5570   op_cost(0);
 5571   format %{ %}
 5572   interface(REG_INTER);
 5573 %}
 5574 
 5575 operand vRegD_V21()
 5576 %{
 5577   constraint(ALLOC_IN_RC(v21_reg));
 5578   match(RegD);
 5579   op_cost(0);
 5580   format %{ %}
 5581   interface(REG_INTER);
 5582 %}
 5583 
 5584 operand vRegD_V22()
 5585 %{
 5586   constraint(ALLOC_IN_RC(v22_reg));
 5587   match(RegD);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 operand vRegD_V23()
 5594 %{
 5595   constraint(ALLOC_IN_RC(v23_reg));
 5596   match(RegD);
 5597   op_cost(0);
 5598   format %{ %}
 5599   interface(REG_INTER);
 5600 %}
 5601 
 5602 operand vRegD_V24()
 5603 %{
 5604   constraint(ALLOC_IN_RC(v24_reg));
 5605   match(RegD);
 5606   op_cost(0);
 5607   format %{ %}
 5608   interface(REG_INTER);
 5609 %}
 5610 
 5611 operand vRegD_V25()
 5612 %{
 5613   constraint(ALLOC_IN_RC(v25_reg));
 5614   match(RegD);
 5615   op_cost(0);
 5616   format %{ %}
 5617   interface(REG_INTER);
 5618 %}
 5619 
 5620 operand vRegD_V26()
 5621 %{
 5622   constraint(ALLOC_IN_RC(v26_reg));
 5623   match(RegD);
 5624   op_cost(0);
 5625   format %{ %}
 5626   interface(REG_INTER);
 5627 %}
 5628 
 5629 operand vRegD_V27()
 5630 %{
 5631   constraint(ALLOC_IN_RC(v27_reg));
 5632   match(RegD);
 5633   op_cost(0);
 5634   format %{ %}
 5635   interface(REG_INTER);
 5636 %}
 5637 
 5638 operand vRegD_V28()
 5639 %{
 5640   constraint(ALLOC_IN_RC(v28_reg));
 5641   match(RegD);
 5642   op_cost(0);
 5643   format %{ %}
 5644   interface(REG_INTER);
 5645 %}
 5646 
 5647 operand vRegD_V29()
 5648 %{
 5649   constraint(ALLOC_IN_RC(v29_reg));
 5650   match(RegD);
 5651   op_cost(0);
 5652   format %{ %}
 5653   interface(REG_INTER);
 5654 %}
 5655 
 5656 operand vRegD_V30()
 5657 %{
 5658   constraint(ALLOC_IN_RC(v30_reg));
 5659   match(RegD);
 5660   op_cost(0);
 5661   format %{ %}
 5662   interface(REG_INTER);
 5663 %}
 5664 
 5665 operand vRegD_V31()
 5666 %{
 5667   constraint(ALLOC_IN_RC(v31_reg));
 5668   match(RegD);
 5669   op_cost(0);
 5670   format %{ %}
 5671   interface(REG_INTER);
 5672 %}
 5673 
 5674 operand pReg()
 5675 %{
 5676   constraint(ALLOC_IN_RC(pr_reg));
 5677   match(RegVectMask);
 5678   match(pRegGov);
 5679   op_cost(0);
 5680   format %{ %}
 5681   interface(REG_INTER);
 5682 %}
 5683 
 5684 operand pRegGov()
 5685 %{
 5686   constraint(ALLOC_IN_RC(gov_pr));
 5687   match(RegVectMask);
 5688   match(pReg);
 5689   op_cost(0);
 5690   format %{ %}
 5691   interface(REG_INTER);
 5692 %}
 5693 
 5694 operand pRegGov_P0()
 5695 %{
 5696   constraint(ALLOC_IN_RC(p0_reg));
 5697   match(RegVectMask);
 5698   op_cost(0);
 5699   format %{ %}
 5700   interface(REG_INTER);
 5701 %}
 5702 
 5703 operand pRegGov_P1()
 5704 %{
 5705   constraint(ALLOC_IN_RC(p1_reg));
 5706   match(RegVectMask);
 5707   op_cost(0);
 5708   format %{ %}
 5709   interface(REG_INTER);
 5710 %}
 5711 
 5712 // Flags register, used as output of signed compare instructions
 5713 
 5714 // note that on AArch64 we also use this register as the output for
 5715 // for floating point compare instructions (CmpF CmpD). this ensures
 5716 // that ordered inequality tests use GT, GE, LT or LE none of which
 5717 // pass through cases where the result is unordered i.e. one or both
 5718 // inputs to the compare is a NaN. this means that the ideal code can
 5719 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5720 // (where the comparison should always fail). EQ and NE tests are
 5721 // always generated in ideal code so that unordered folds into the NE
 5722 // case, matching the behaviour of AArch64 NE.
 5723 //
 5724 // This differs from x86 where the outputs of FP compares use a
 5725 // special FP flags registers and where compares based on this
 5726 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5727 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5728 // to explicitly handle the unordered case in branches. x86 also has
 5729 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5730 
 5731 operand rFlagsReg()
 5732 %{
 5733   constraint(ALLOC_IN_RC(int_flags));
 5734   match(RegFlags);
 5735 
 5736   op_cost(0);
 5737   format %{ "RFLAGS" %}
 5738   interface(REG_INTER);
 5739 %}
 5740 
 5741 // Flags register, used as output of unsigned compare instructions
 5742 operand rFlagsRegU()
 5743 %{
 5744   constraint(ALLOC_IN_RC(int_flags));
 5745   match(RegFlags);
 5746 
 5747   op_cost(0);
 5748   format %{ "RFLAGSU" %}
 5749   interface(REG_INTER);
 5750 %}
 5751 
 5752 // Special Registers
 5753 
 5754 // Method Register
 5755 operand inline_cache_RegP(iRegP reg)
 5756 %{
 5757   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5758   match(reg);
 5759   match(iRegPNoSp);
 5760   op_cost(0);
 5761   format %{ %}
 5762   interface(REG_INTER);
 5763 %}
 5764 
 5765 // Thread Register
 5766 operand thread_RegP(iRegP reg)
 5767 %{
 5768   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5769   match(reg);
 5770   op_cost(0);
 5771   format %{ %}
 5772   interface(REG_INTER);
 5773 %}
 5774 
 5775 operand lr_RegP(iRegP reg)
 5776 %{
 5777   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5778   match(reg);
 5779   op_cost(0);
 5780   format %{ %}
 5781   interface(REG_INTER);
 5782 %}
 5783 
 5784 //----------Memory Operands----------------------------------------------------
 5785 
 5786 operand indirect(iRegP reg)
 5787 %{
 5788   constraint(ALLOC_IN_RC(ptr_reg));
 5789   match(reg);
 5790   op_cost(0);
 5791   format %{ "[$reg]" %}
 5792   interface(MEMORY_INTER) %{
 5793     base($reg);
 5794     index(0xffffffff);
 5795     scale(0x0);
 5796     disp(0x0);
 5797   %}
 5798 %}
 5799 
 5800 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5801 %{
 5802   constraint(ALLOC_IN_RC(ptr_reg));
 5803   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5804   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5805   op_cost(0);
 5806   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5807   interface(MEMORY_INTER) %{
 5808     base($reg);
 5809     index($ireg);
 5810     scale($scale);
 5811     disp(0x0);
 5812   %}
 5813 %}
 5814 
 5815 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5816 %{
 5817   constraint(ALLOC_IN_RC(ptr_reg));
 5818   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5819   match(AddP reg (LShiftL lreg scale));
 5820   op_cost(0);
 5821   format %{ "$reg, $lreg lsl($scale)" %}
 5822   interface(MEMORY_INTER) %{
 5823     base($reg);
 5824     index($lreg);
 5825     scale($scale);
 5826     disp(0x0);
 5827   %}
 5828 %}
 5829 
 5830 operand indIndexI2L(iRegP reg, iRegI ireg)
 5831 %{
 5832   constraint(ALLOC_IN_RC(ptr_reg));
 5833   match(AddP reg (ConvI2L ireg));
 5834   op_cost(0);
 5835   format %{ "$reg, $ireg, 0, I2L" %}
 5836   interface(MEMORY_INTER) %{
 5837     base($reg);
 5838     index($ireg);
 5839     scale(0x0);
 5840     disp(0x0);
 5841   %}
 5842 %}
 5843 
 5844 operand indIndex(iRegP reg, iRegL lreg)
 5845 %{
 5846   constraint(ALLOC_IN_RC(ptr_reg));
 5847   match(AddP reg lreg);
 5848   op_cost(0);
 5849   format %{ "$reg, $lreg" %}
 5850   interface(MEMORY_INTER) %{
 5851     base($reg);
 5852     index($lreg);
 5853     scale(0x0);
 5854     disp(0x0);
 5855   %}
 5856 %}
 5857 
 5858 operand indOffI(iRegP reg, immIOffset off)
 5859 %{
 5860   constraint(ALLOC_IN_RC(ptr_reg));
 5861   match(AddP reg off);
 5862   op_cost(0);
 5863   format %{ "[$reg, $off]" %}
 5864   interface(MEMORY_INTER) %{
 5865     base($reg);
 5866     index(0xffffffff);
 5867     scale(0x0);
 5868     disp($off);
 5869   %}
 5870 %}
 5871 
 5872 operand indOffI1(iRegP reg, immIOffset1 off)
 5873 %{
 5874   constraint(ALLOC_IN_RC(ptr_reg));
 5875   match(AddP reg off);
 5876   op_cost(0);
 5877   format %{ "[$reg, $off]" %}
 5878   interface(MEMORY_INTER) %{
 5879     base($reg);
 5880     index(0xffffffff);
 5881     scale(0x0);
 5882     disp($off);
 5883   %}
 5884 %}
 5885 
 5886 operand indOffI2(iRegP reg, immIOffset2 off)
 5887 %{
 5888   constraint(ALLOC_IN_RC(ptr_reg));
 5889   match(AddP reg off);
 5890   op_cost(0);
 5891   format %{ "[$reg, $off]" %}
 5892   interface(MEMORY_INTER) %{
 5893     base($reg);
 5894     index(0xffffffff);
 5895     scale(0x0);
 5896     disp($off);
 5897   %}
 5898 %}
 5899 
 5900 operand indOffI4(iRegP reg, immIOffset4 off)
 5901 %{
 5902   constraint(ALLOC_IN_RC(ptr_reg));
 5903   match(AddP reg off);
 5904   op_cost(0);
 5905   format %{ "[$reg, $off]" %}
 5906   interface(MEMORY_INTER) %{
 5907     base($reg);
 5908     index(0xffffffff);
 5909     scale(0x0);
 5910     disp($off);
 5911   %}
 5912 %}
 5913 
 5914 operand indOffI8(iRegP reg, immIOffset8 off)
 5915 %{
 5916   constraint(ALLOC_IN_RC(ptr_reg));
 5917   match(AddP reg off);
 5918   op_cost(0);
 5919   format %{ "[$reg, $off]" %}
 5920   interface(MEMORY_INTER) %{
 5921     base($reg);
 5922     index(0xffffffff);
 5923     scale(0x0);
 5924     disp($off);
 5925   %}
 5926 %}
 5927 
 5928 operand indOffI16(iRegP reg, immIOffset16 off)
 5929 %{
 5930   constraint(ALLOC_IN_RC(ptr_reg));
 5931   match(AddP reg off);
 5932   op_cost(0);
 5933   format %{ "[$reg, $off]" %}
 5934   interface(MEMORY_INTER) %{
 5935     base($reg);
 5936     index(0xffffffff);
 5937     scale(0x0);
 5938     disp($off);
 5939   %}
 5940 %}
 5941 
 5942 operand indOffL(iRegP reg, immLoffset off)
 5943 %{
 5944   constraint(ALLOC_IN_RC(ptr_reg));
 5945   match(AddP reg off);
 5946   op_cost(0);
 5947   format %{ "[$reg, $off]" %}
 5948   interface(MEMORY_INTER) %{
 5949     base($reg);
 5950     index(0xffffffff);
 5951     scale(0x0);
 5952     disp($off);
 5953   %}
 5954 %}
 5955 
 5956 operand indOffL1(iRegP reg, immLoffset1 off)
 5957 %{
 5958   constraint(ALLOC_IN_RC(ptr_reg));
 5959   match(AddP reg off);
 5960   op_cost(0);
 5961   format %{ "[$reg, $off]" %}
 5962   interface(MEMORY_INTER) %{
 5963     base($reg);
 5964     index(0xffffffff);
 5965     scale(0x0);
 5966     disp($off);
 5967   %}
 5968 %}
 5969 
 5970 operand indOffL2(iRegP reg, immLoffset2 off)
 5971 %{
 5972   constraint(ALLOC_IN_RC(ptr_reg));
 5973   match(AddP reg off);
 5974   op_cost(0);
 5975   format %{ "[$reg, $off]" %}
 5976   interface(MEMORY_INTER) %{
 5977     base($reg);
 5978     index(0xffffffff);
 5979     scale(0x0);
 5980     disp($off);
 5981   %}
 5982 %}
 5983 
 5984 operand indOffL4(iRegP reg, immLoffset4 off)
 5985 %{
 5986   constraint(ALLOC_IN_RC(ptr_reg));
 5987   match(AddP reg off);
 5988   op_cost(0);
 5989   format %{ "[$reg, $off]" %}
 5990   interface(MEMORY_INTER) %{
 5991     base($reg);
 5992     index(0xffffffff);
 5993     scale(0x0);
 5994     disp($off);
 5995   %}
 5996 %}
 5997 
 5998 operand indOffL8(iRegP reg, immLoffset8 off)
 5999 %{
 6000   constraint(ALLOC_IN_RC(ptr_reg));
 6001   match(AddP reg off);
 6002   op_cost(0);
 6003   format %{ "[$reg, $off]" %}
 6004   interface(MEMORY_INTER) %{
 6005     base($reg);
 6006     index(0xffffffff);
 6007     scale(0x0);
 6008     disp($off);
 6009   %}
 6010 %}
 6011 
 6012 operand indOffL16(iRegP reg, immLoffset16 off)
 6013 %{
 6014   constraint(ALLOC_IN_RC(ptr_reg));
 6015   match(AddP reg off);
 6016   op_cost(0);
 6017   format %{ "[$reg, $off]" %}
 6018   interface(MEMORY_INTER) %{
 6019     base($reg);
 6020     index(0xffffffff);
 6021     scale(0x0);
 6022     disp($off);
 6023   %}
 6024 %}
 6025 
 6026 operand indirectN(iRegN reg)
 6027 %{
 6028   predicate(CompressedOops::shift() == 0);
 6029   constraint(ALLOC_IN_RC(ptr_reg));
 6030   match(DecodeN reg);
 6031   op_cost(0);
 6032   format %{ "[$reg]\t# narrow" %}
 6033   interface(MEMORY_INTER) %{
 6034     base($reg);
 6035     index(0xffffffff);
 6036     scale(0x0);
 6037     disp(0x0);
 6038   %}
 6039 %}
 6040 
 6041 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 6042 %{
 6043   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6044   constraint(ALLOC_IN_RC(ptr_reg));
 6045   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 6046   op_cost(0);
 6047   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 6048   interface(MEMORY_INTER) %{
 6049     base($reg);
 6050     index($ireg);
 6051     scale($scale);
 6052     disp(0x0);
 6053   %}
 6054 %}
 6055 
 6056 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 6057 %{
 6058   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 6059   constraint(ALLOC_IN_RC(ptr_reg));
 6060   match(AddP (DecodeN reg) (LShiftL lreg scale));
 6061   op_cost(0);
 6062   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 6063   interface(MEMORY_INTER) %{
 6064     base($reg);
 6065     index($lreg);
 6066     scale($scale);
 6067     disp(0x0);
 6068   %}
 6069 %}
 6070 
 6071 operand indIndexI2LN(iRegN reg, iRegI ireg)
 6072 %{
 6073   predicate(CompressedOops::shift() == 0);
 6074   constraint(ALLOC_IN_RC(ptr_reg));
 6075   match(AddP (DecodeN reg) (ConvI2L ireg));
 6076   op_cost(0);
 6077   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 6078   interface(MEMORY_INTER) %{
 6079     base($reg);
 6080     index($ireg);
 6081     scale(0x0);
 6082     disp(0x0);
 6083   %}
 6084 %}
 6085 
 6086 operand indIndexN(iRegN reg, iRegL lreg)
 6087 %{
 6088   predicate(CompressedOops::shift() == 0);
 6089   constraint(ALLOC_IN_RC(ptr_reg));
 6090   match(AddP (DecodeN reg) lreg);
 6091   op_cost(0);
 6092   format %{ "$reg, $lreg\t# narrow" %}
 6093   interface(MEMORY_INTER) %{
 6094     base($reg);
 6095     index($lreg);
 6096     scale(0x0);
 6097     disp(0x0);
 6098   %}
 6099 %}
 6100 
 6101 operand indOffIN(iRegN reg, immIOffset off)
 6102 %{
 6103   predicate(CompressedOops::shift() == 0);
 6104   constraint(ALLOC_IN_RC(ptr_reg));
 6105   match(AddP (DecodeN reg) off);
 6106   op_cost(0);
 6107   format %{ "[$reg, $off]\t# narrow" %}
 6108   interface(MEMORY_INTER) %{
 6109     base($reg);
 6110     index(0xffffffff);
 6111     scale(0x0);
 6112     disp($off);
 6113   %}
 6114 %}
 6115 
 6116 operand indOffLN(iRegN reg, immLoffset off)
 6117 %{
 6118   predicate(CompressedOops::shift() == 0);
 6119   constraint(ALLOC_IN_RC(ptr_reg));
 6120   match(AddP (DecodeN reg) off);
 6121   op_cost(0);
 6122   format %{ "[$reg, $off]\t# narrow" %}
 6123   interface(MEMORY_INTER) %{
 6124     base($reg);
 6125     index(0xffffffff);
 6126     scale(0x0);
 6127     disp($off);
 6128   %}
 6129 %}
 6130 
 6131 
 6132 
 6133 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6134 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6135 %{
 6136   constraint(ALLOC_IN_RC(ptr_reg));
 6137   match(AddP reg off);
 6138   op_cost(0);
 6139   format %{ "[$reg, $off]" %}
 6140   interface(MEMORY_INTER) %{
 6141     base($reg);
 6142     index(0xffffffff);
 6143     scale(0x0);
 6144     disp($off);
 6145   %}
 6146 %}
 6147 
 6148 //----------Special Memory Operands--------------------------------------------
 6149 // Stack Slot Operand - This operand is used for loading and storing temporary
 6150 //                      values on the stack where a match requires a value to
 6151 //                      flow through memory.
 6152 operand stackSlotP(sRegP reg)
 6153 %{
 6154   constraint(ALLOC_IN_RC(stack_slots));
 6155   op_cost(100);
 6156   // No match rule because this operand is only generated in matching
 6157   // match(RegP);
 6158   format %{ "[$reg]" %}
 6159   interface(MEMORY_INTER) %{
 6160     base(0x1e);  // RSP
 6161     index(0x0);  // No Index
 6162     scale(0x0);  // No Scale
 6163     disp($reg);  // Stack Offset
 6164   %}
 6165 %}
 6166 
 6167 operand stackSlotI(sRegI reg)
 6168 %{
 6169   constraint(ALLOC_IN_RC(stack_slots));
 6170   // No match rule because this operand is only generated in matching
 6171   // match(RegI);
 6172   format %{ "[$reg]" %}
 6173   interface(MEMORY_INTER) %{
 6174     base(0x1e);  // RSP
 6175     index(0x0);  // No Index
 6176     scale(0x0);  // No Scale
 6177     disp($reg);  // Stack Offset
 6178   %}
 6179 %}
 6180 
 6181 operand stackSlotF(sRegF reg)
 6182 %{
 6183   constraint(ALLOC_IN_RC(stack_slots));
 6184   // No match rule because this operand is only generated in matching
 6185   // match(RegF);
 6186   format %{ "[$reg]" %}
 6187   interface(MEMORY_INTER) %{
 6188     base(0x1e);  // RSP
 6189     index(0x0);  // No Index
 6190     scale(0x0);  // No Scale
 6191     disp($reg);  // Stack Offset
 6192   %}
 6193 %}
 6194 
 6195 operand stackSlotD(sRegD reg)
 6196 %{
 6197   constraint(ALLOC_IN_RC(stack_slots));
 6198   // No match rule because this operand is only generated in matching
 6199   // match(RegD);
 6200   format %{ "[$reg]" %}
 6201   interface(MEMORY_INTER) %{
 6202     base(0x1e);  // RSP
 6203     index(0x0);  // No Index
 6204     scale(0x0);  // No Scale
 6205     disp($reg);  // Stack Offset
 6206   %}
 6207 %}
 6208 
 6209 operand stackSlotL(sRegL reg)
 6210 %{
 6211   constraint(ALLOC_IN_RC(stack_slots));
 6212   // No match rule because this operand is only generated in matching
 6213   // match(RegL);
 6214   format %{ "[$reg]" %}
 6215   interface(MEMORY_INTER) %{
 6216     base(0x1e);  // RSP
 6217     index(0x0);  // No Index
 6218     scale(0x0);  // No Scale
 6219     disp($reg);  // Stack Offset
 6220   %}
 6221 %}
 6222 
 6223 // Operands for expressing Control Flow
 6224 // NOTE: Label is a predefined operand which should not be redefined in
 6225 //       the AD file. It is generically handled within the ADLC.
 6226 
 6227 //----------Conditional Branch Operands----------------------------------------
 6228 // Comparison Op  - This is the operation of the comparison, and is limited to
 6229 //                  the following set of codes:
 6230 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6231 //
 6232 // Other attributes of the comparison, such as unsignedness, are specified
 6233 // by the comparison instruction that sets a condition code flags register.
 6234 // That result is represented by a flags operand whose subtype is appropriate
 6235 // to the unsignedness (etc.) of the comparison.
 6236 //
 6237 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6238 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6239 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6240 
 6241 // used for signed integral comparisons and fp comparisons
 6242 
 6243 operand cmpOp()
 6244 %{
 6245   match(Bool);
 6246 
 6247   format %{ "" %}
 6248   interface(COND_INTER) %{
 6249     equal(0x0, "eq");
 6250     not_equal(0x1, "ne");
 6251     less(0xb, "lt");
 6252     greater_equal(0xa, "ge");
 6253     less_equal(0xd, "le");
 6254     greater(0xc, "gt");
 6255     overflow(0x6, "vs");
 6256     no_overflow(0x7, "vc");
 6257   %}
 6258 %}
 6259 
 6260 // used for unsigned integral comparisons
 6261 
 6262 operand cmpOpU()
 6263 %{
 6264   match(Bool);
 6265 
 6266   format %{ "" %}
 6267   interface(COND_INTER) %{
 6268     equal(0x0, "eq");
 6269     not_equal(0x1, "ne");
 6270     less(0x3, "lo");
 6271     greater_equal(0x2, "hs");
 6272     less_equal(0x9, "ls");
 6273     greater(0x8, "hi");
 6274     overflow(0x6, "vs");
 6275     no_overflow(0x7, "vc");
 6276   %}
 6277 %}
 6278 
 6279 // used for certain integral comparisons which can be
 6280 // converted to cbxx or tbxx instructions
 6281 
 6282 operand cmpOpEqNe()
 6283 %{
 6284   match(Bool);
 6285   op_cost(0);
 6286   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6287             || n->as_Bool()->_test._test == BoolTest::eq);
 6288 
 6289   format %{ "" %}
 6290   interface(COND_INTER) %{
 6291     equal(0x0, "eq");
 6292     not_equal(0x1, "ne");
 6293     less(0xb, "lt");
 6294     greater_equal(0xa, "ge");
 6295     less_equal(0xd, "le");
 6296     greater(0xc, "gt");
 6297     overflow(0x6, "vs");
 6298     no_overflow(0x7, "vc");
 6299   %}
 6300 %}
 6301 
 6302 // used for certain integral comparisons which can be
 6303 // converted to cbxx or tbxx instructions
 6304 
 6305 operand cmpOpLtGe()
 6306 %{
 6307   match(Bool);
 6308   op_cost(0);
 6309 
 6310   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6311             || n->as_Bool()->_test._test == BoolTest::ge);
 6312 
 6313   format %{ "" %}
 6314   interface(COND_INTER) %{
 6315     equal(0x0, "eq");
 6316     not_equal(0x1, "ne");
 6317     less(0xb, "lt");
 6318     greater_equal(0xa, "ge");
 6319     less_equal(0xd, "le");
 6320     greater(0xc, "gt");
 6321     overflow(0x6, "vs");
 6322     no_overflow(0x7, "vc");
 6323   %}
 6324 %}
 6325 
 6326 // used for certain unsigned integral comparisons which can be
 6327 // converted to cbxx or tbxx instructions
 6328 
 6329 operand cmpOpUEqNeLtGe()
 6330 %{
 6331   match(Bool);
 6332   op_cost(0);
 6333 
 6334   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6335             || n->as_Bool()->_test._test == BoolTest::ne
 6336             || n->as_Bool()->_test._test == BoolTest::lt
 6337             || n->as_Bool()->_test._test == BoolTest::ge);
 6338 
 6339   format %{ "" %}
 6340   interface(COND_INTER) %{
 6341     equal(0x0, "eq");
 6342     not_equal(0x1, "ne");
 6343     less(0xb, "lt");
 6344     greater_equal(0xa, "ge");
 6345     less_equal(0xd, "le");
 6346     greater(0xc, "gt");
 6347     overflow(0x6, "vs");
 6348     no_overflow(0x7, "vc");
 6349   %}
 6350 %}
 6351 
 6352 // Special operand allowing long args to int ops to be truncated for free
 6353 
 6354 operand iRegL2I(iRegL reg) %{
 6355 
 6356   op_cost(0);
 6357 
 6358   match(ConvL2I reg);
 6359 
 6360   format %{ "l2i($reg)" %}
 6361 
 6362   interface(REG_INTER)
 6363 %}
 6364 
 6365 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6366 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6367 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6368 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6369 
 6370 //----------OPERAND CLASSES----------------------------------------------------
 6371 // Operand Classes are groups of operands that are used as to simplify
 6372 // instruction definitions by not requiring the AD writer to specify
 6373 // separate instructions for every form of operand when the
 6374 // instruction accepts multiple operand types with the same basic
 6375 // encoding and format. The classic case of this is memory operands.
 6376 
 6377 // memory is used to define read/write location for load/store
 6378 // instruction defs. we can turn a memory op into an Address
 6379 
 6380 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6381                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6382 
 6383 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6384                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6385 
 6386 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6387                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6388 
 6389 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6390                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6391 
 6392 // All of the memory operands. For the pipeline description.
 6393 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6394                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6395                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6396 
 6397 
 6398 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6399 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6400 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6401 // can be elided because the 32-bit instruction will just employ the
 6402 // lower 32 bits anyway.
 6403 //
 6404 // n.b. this does not elide all L2I conversions. if the truncated
 6405 // value is consumed by more than one operation then the ConvL2I
 6406 // cannot be bundled into the consuming nodes so an l2i gets planted
 6407 // (actually a movw $dst $src) and the downstream instructions consume
 6408 // the result of the l2i as an iRegI input. That's a shame since the
 6409 // movw is actually redundant but its not too costly.
 6410 
 6411 opclass iRegIorL2I(iRegI, iRegL2I);
 6412 
 6413 //----------PIPELINE-----------------------------------------------------------
 6414 // Rules which define the behavior of the target architectures pipeline.
 6415 
 6416 // For specific pipelines, eg A53, define the stages of that pipeline
 6417 //pipe_desc(ISS, EX1, EX2, WR);
 6418 #define ISS S0
 6419 #define EX1 S1
 6420 #define EX2 S2
 6421 #define WR  S3
 6422 
 6423 // Integer ALU reg operation
 6424 pipeline %{
 6425 
 6426 attributes %{
 6427   // ARM instructions are of fixed length
 6428   fixed_size_instructions;        // Fixed size instructions TODO does
 6429   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6430   // ARM instructions come in 32-bit word units
 6431   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6432   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6433   instruction_fetch_units = 1;       // of 64 bytes
 6434 
 6435   // List of nop instructions
 6436   nops( MachNop );
 6437 %}
 6438 
 6439 // We don't use an actual pipeline model so don't care about resources
 6440 // or description. we do use pipeline classes to introduce fixed
 6441 // latencies
 6442 
 6443 //----------RESOURCES----------------------------------------------------------
 6444 // Resources are the functional units available to the machine
 6445 
 6446 resources( INS0, INS1, INS01 = INS0 | INS1,
 6447            ALU0, ALU1, ALU = ALU0 | ALU1,
 6448            MAC,
 6449            DIV,
 6450            BRANCH,
 6451            LDST,
 6452            NEON_FP);
 6453 
 6454 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6455 // Pipeline Description specifies the stages in the machine's pipeline
 6456 
 6457 // Define the pipeline as a generic 6 stage pipeline
 6458 pipe_desc(S0, S1, S2, S3, S4, S5);
 6459 
 6460 //----------PIPELINE CLASSES---------------------------------------------------
 6461 // Pipeline Classes describe the stages in which input and output are
 6462 // referenced by the hardware pipeline.
 6463 
 6464 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6465 %{
 6466   single_instruction;
 6467   src1   : S1(read);
 6468   src2   : S2(read);
 6469   dst    : S5(write);
 6470   INS01  : ISS;
 6471   NEON_FP : S5;
 6472 %}
 6473 
 6474 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6475 %{
 6476   single_instruction;
 6477   src1   : S1(read);
 6478   src2   : S2(read);
 6479   dst    : S5(write);
 6480   INS01  : ISS;
 6481   NEON_FP : S5;
 6482 %}
 6483 
 6484 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6485 %{
 6486   single_instruction;
 6487   src    : S1(read);
 6488   dst    : S5(write);
 6489   INS01  : ISS;
 6490   NEON_FP : S5;
 6491 %}
 6492 
 6493 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6494 %{
 6495   single_instruction;
 6496   src    : S1(read);
 6497   dst    : S5(write);
 6498   INS01  : ISS;
 6499   NEON_FP : S5;
 6500 %}
 6501 
 6502 pipe_class fp_d2f(vRegF dst, vRegD src)
 6503 %{
 6504   single_instruction;
 6505   src    : S1(read);
 6506   dst    : S5(write);
 6507   INS01  : ISS;
 6508   NEON_FP : S5;
 6509 %}
 6510 
 6511 pipe_class fp_f2d(vRegD dst, vRegF src)
 6512 %{
 6513   single_instruction;
 6514   src    : S1(read);
 6515   dst    : S5(write);
 6516   INS01  : ISS;
 6517   NEON_FP : S5;
 6518 %}
 6519 
 6520 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6521 %{
 6522   single_instruction;
 6523   src    : S1(read);
 6524   dst    : S5(write);
 6525   INS01  : ISS;
 6526   NEON_FP : S5;
 6527 %}
 6528 
 6529 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6530 %{
 6531   single_instruction;
 6532   src    : S1(read);
 6533   dst    : S5(write);
 6534   INS01  : ISS;
 6535   NEON_FP : S5;
 6536 %}
 6537 
 6538 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6539 %{
 6540   single_instruction;
 6541   src    : S1(read);
 6542   dst    : S5(write);
 6543   INS01  : ISS;
 6544   NEON_FP : S5;
 6545 %}
 6546 
 6547 pipe_class fp_l2f(vRegF dst, iRegL src)
 6548 %{
 6549   single_instruction;
 6550   src    : S1(read);
 6551   dst    : S5(write);
 6552   INS01  : ISS;
 6553   NEON_FP : S5;
 6554 %}
 6555 
 6556 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6557 %{
 6558   single_instruction;
 6559   src    : S1(read);
 6560   dst    : S5(write);
 6561   INS01  : ISS;
 6562   NEON_FP : S5;
 6563 %}
 6564 
 6565 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6566 %{
 6567   single_instruction;
 6568   src    : S1(read);
 6569   dst    : S5(write);
 6570   INS01  : ISS;
 6571   NEON_FP : S5;
 6572 %}
 6573 
 6574 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6575 %{
 6576   single_instruction;
 6577   src    : S1(read);
 6578   dst    : S5(write);
 6579   INS01  : ISS;
 6580   NEON_FP : S5;
 6581 %}
 6582 
 6583 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6584 %{
 6585   single_instruction;
 6586   src    : S1(read);
 6587   dst    : S5(write);
 6588   INS01  : ISS;
 6589   NEON_FP : S5;
 6590 %}
 6591 
 6592 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6593 %{
 6594   single_instruction;
 6595   src1   : S1(read);
 6596   src2   : S2(read);
 6597   dst    : S5(write);
 6598   INS0   : ISS;
 6599   NEON_FP : S5;
 6600 %}
 6601 
 6602 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6603 %{
 6604   single_instruction;
 6605   src1   : S1(read);
 6606   src2   : S2(read);
 6607   dst    : S5(write);
 6608   INS0   : ISS;
 6609   NEON_FP : S5;
 6610 %}
 6611 
 6612 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6613 %{
 6614   single_instruction;
 6615   cr     : S1(read);
 6616   src1   : S1(read);
 6617   src2   : S1(read);
 6618   dst    : S3(write);
 6619   INS01  : ISS;
 6620   NEON_FP : S3;
 6621 %}
 6622 
 6623 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6624 %{
 6625   single_instruction;
 6626   cr     : S1(read);
 6627   src1   : S1(read);
 6628   src2   : S1(read);
 6629   dst    : S3(write);
 6630   INS01  : ISS;
 6631   NEON_FP : S3;
 6632 %}
 6633 
 6634 pipe_class fp_imm_s(vRegF dst)
 6635 %{
 6636   single_instruction;
 6637   dst    : S3(write);
 6638   INS01  : ISS;
 6639   NEON_FP : S3;
 6640 %}
 6641 
 6642 pipe_class fp_imm_d(vRegD dst)
 6643 %{
 6644   single_instruction;
 6645   dst    : S3(write);
 6646   INS01  : ISS;
 6647   NEON_FP : S3;
 6648 %}
 6649 
 6650 pipe_class fp_load_constant_s(vRegF dst)
 6651 %{
 6652   single_instruction;
 6653   dst    : S4(write);
 6654   INS01  : ISS;
 6655   NEON_FP : S4;
 6656 %}
 6657 
 6658 pipe_class fp_load_constant_d(vRegD dst)
 6659 %{
 6660   single_instruction;
 6661   dst    : S4(write);
 6662   INS01  : ISS;
 6663   NEON_FP : S4;
 6664 %}
 6665 
 6666 //------- Integer ALU operations --------------------------
 6667 
 6668 // Integer ALU reg-reg operation
 6669 // Operands needed in EX1, result generated in EX2
 6670 // Eg.  ADD     x0, x1, x2
 6671 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6672 %{
 6673   single_instruction;
 6674   dst    : EX2(write);
 6675   src1   : EX1(read);
 6676   src2   : EX1(read);
 6677   INS01  : ISS; // Dual issue as instruction 0 or 1
 6678   ALU    : EX2;
 6679 %}
 6680 
 6681 // Integer ALU reg-reg operation with constant shift
 6682 // Shifted register must be available in LATE_ISS instead of EX1
 6683 // Eg.  ADD     x0, x1, x2, LSL #2
 6684 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6685 %{
 6686   single_instruction;
 6687   dst    : EX2(write);
 6688   src1   : EX1(read);
 6689   src2   : ISS(read);
 6690   INS01  : ISS;
 6691   ALU    : EX2;
 6692 %}
 6693 
 6694 // Integer ALU reg operation with constant shift
 6695 // Eg.  LSL     x0, x1, #shift
 6696 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6697 %{
 6698   single_instruction;
 6699   dst    : EX2(write);
 6700   src1   : ISS(read);
 6701   INS01  : ISS;
 6702   ALU    : EX2;
 6703 %}
 6704 
 6705 // Integer ALU reg-reg operation with variable shift
 6706 // Both operands must be available in LATE_ISS instead of EX1
 6707 // Result is available in EX1 instead of EX2
 6708 // Eg.  LSLV    x0, x1, x2
 6709 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6710 %{
 6711   single_instruction;
 6712   dst    : EX1(write);
 6713   src1   : ISS(read);
 6714   src2   : ISS(read);
 6715   INS01  : ISS;
 6716   ALU    : EX1;
 6717 %}
 6718 
 6719 // Integer ALU reg-reg operation with extract
 6720 // As for _vshift above, but result generated in EX2
 6721 // Eg.  EXTR    x0, x1, x2, #N
 6722 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6723 %{
 6724   single_instruction;
 6725   dst    : EX2(write);
 6726   src1   : ISS(read);
 6727   src2   : ISS(read);
 6728   INS1   : ISS; // Can only dual issue as Instruction 1
 6729   ALU    : EX1;
 6730 %}
 6731 
 6732 // Integer ALU reg operation
 6733 // Eg.  NEG     x0, x1
 6734 pipe_class ialu_reg(iRegI dst, iRegI src)
 6735 %{
 6736   single_instruction;
 6737   dst    : EX2(write);
 6738   src    : EX1(read);
 6739   INS01  : ISS;
 6740   ALU    : EX2;
 6741 %}
 6742 
 6743 // Integer ALU reg mmediate operation
 6744 // Eg.  ADD     x0, x1, #N
 6745 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6746 %{
 6747   single_instruction;
 6748   dst    : EX2(write);
 6749   src1   : EX1(read);
 6750   INS01  : ISS;
 6751   ALU    : EX2;
 6752 %}
 6753 
 6754 // Integer ALU immediate operation (no source operands)
 6755 // Eg.  MOV     x0, #N
 6756 pipe_class ialu_imm(iRegI dst)
 6757 %{
 6758   single_instruction;
 6759   dst    : EX1(write);
 6760   INS01  : ISS;
 6761   ALU    : EX1;
 6762 %}
 6763 
 6764 //------- Compare operation -------------------------------
 6765 
 6766 // Compare reg-reg
 6767 // Eg.  CMP     x0, x1
 6768 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6769 %{
 6770   single_instruction;
 6771 //  fixed_latency(16);
 6772   cr     : EX2(write);
 6773   op1    : EX1(read);
 6774   op2    : EX1(read);
 6775   INS01  : ISS;
 6776   ALU    : EX2;
 6777 %}
 6778 
 6779 // Compare reg-reg
 6780 // Eg.  CMP     x0, #N
 6781 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6782 %{
 6783   single_instruction;
 6784 //  fixed_latency(16);
 6785   cr     : EX2(write);
 6786   op1    : EX1(read);
 6787   INS01  : ISS;
 6788   ALU    : EX2;
 6789 %}
 6790 
 6791 //------- Conditional instructions ------------------------
 6792 
 6793 // Conditional no operands
 6794 // Eg.  CSINC   x0, zr, zr, <cond>
 6795 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6796 %{
 6797   single_instruction;
 6798   cr     : EX1(read);
 6799   dst    : EX2(write);
 6800   INS01  : ISS;
 6801   ALU    : EX2;
 6802 %}
 6803 
 6804 // Conditional 2 operand
 6805 // EG.  CSEL    X0, X1, X2, <cond>
 6806 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6807 %{
 6808   single_instruction;
 6809   cr     : EX1(read);
 6810   src1   : EX1(read);
 6811   src2   : EX1(read);
 6812   dst    : EX2(write);
 6813   INS01  : ISS;
 6814   ALU    : EX2;
 6815 %}
 6816 
 6817 // Conditional 2 operand
 6818 // EG.  CSEL    X0, X1, X2, <cond>
 6819 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6820 %{
 6821   single_instruction;
 6822   cr     : EX1(read);
 6823   src    : EX1(read);
 6824   dst    : EX2(write);
 6825   INS01  : ISS;
 6826   ALU    : EX2;
 6827 %}
 6828 
 6829 //------- Multiply pipeline operations --------------------
 6830 
 6831 // Multiply reg-reg
 6832 // Eg.  MUL     w0, w1, w2
 6833 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6834 %{
 6835   single_instruction;
 6836   dst    : WR(write);
 6837   src1   : ISS(read);
 6838   src2   : ISS(read);
 6839   INS01  : ISS;
 6840   MAC    : WR;
 6841 %}
 6842 
 6843 // Multiply accumulate
 6844 // Eg.  MADD    w0, w1, w2, w3
 6845 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6846 %{
 6847   single_instruction;
 6848   dst    : WR(write);
 6849   src1   : ISS(read);
 6850   src2   : ISS(read);
 6851   src3   : ISS(read);
 6852   INS01  : ISS;
 6853   MAC    : WR;
 6854 %}
 6855 
 6856 // Eg.  MUL     w0, w1, w2
 6857 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6858 %{
 6859   single_instruction;
 6860   fixed_latency(3); // Maximum latency for 64 bit mul
 6861   dst    : WR(write);
 6862   src1   : ISS(read);
 6863   src2   : ISS(read);
 6864   INS01  : ISS;
 6865   MAC    : WR;
 6866 %}
 6867 
 6868 // Multiply accumulate
 6869 // Eg.  MADD    w0, w1, w2, w3
 6870 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6871 %{
 6872   single_instruction;
 6873   fixed_latency(3); // Maximum latency for 64 bit mul
 6874   dst    : WR(write);
 6875   src1   : ISS(read);
 6876   src2   : ISS(read);
 6877   src3   : ISS(read);
 6878   INS01  : ISS;
 6879   MAC    : WR;
 6880 %}
 6881 
 6882 //------- Divide pipeline operations --------------------
 6883 
 6884 // Eg.  SDIV    w0, w1, w2
 6885 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6886 %{
 6887   single_instruction;
 6888   fixed_latency(8); // Maximum latency for 32 bit divide
 6889   dst    : WR(write);
 6890   src1   : ISS(read);
 6891   src2   : ISS(read);
 6892   INS0   : ISS; // Can only dual issue as instruction 0
 6893   DIV    : WR;
 6894 %}
 6895 
 6896 // Eg.  SDIV    x0, x1, x2
 6897 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6898 %{
 6899   single_instruction;
 6900   fixed_latency(16); // Maximum latency for 64 bit divide
 6901   dst    : WR(write);
 6902   src1   : ISS(read);
 6903   src2   : ISS(read);
 6904   INS0   : ISS; // Can only dual issue as instruction 0
 6905   DIV    : WR;
 6906 %}
 6907 
 6908 //------- Load pipeline operations ------------------------
 6909 
 6910 // Load - prefetch
 6911 // Eg.  PFRM    <mem>
 6912 pipe_class iload_prefetch(memory mem)
 6913 %{
 6914   single_instruction;
 6915   mem    : ISS(read);
 6916   INS01  : ISS;
 6917   LDST   : WR;
 6918 %}
 6919 
 6920 // Load - reg, mem
 6921 // Eg.  LDR     x0, <mem>
 6922 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6923 %{
 6924   single_instruction;
 6925   dst    : WR(write);
 6926   mem    : ISS(read);
 6927   INS01  : ISS;
 6928   LDST   : WR;
 6929 %}
 6930 
 6931 // Load - reg, reg
 6932 // Eg.  LDR     x0, [sp, x1]
 6933 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6934 %{
 6935   single_instruction;
 6936   dst    : WR(write);
 6937   src    : ISS(read);
 6938   INS01  : ISS;
 6939   LDST   : WR;
 6940 %}
 6941 
 6942 //------- Store pipeline operations -----------------------
 6943 
 6944 // Store - zr, mem
 6945 // Eg.  STR     zr, <mem>
 6946 pipe_class istore_mem(memory mem)
 6947 %{
 6948   single_instruction;
 6949   mem    : ISS(read);
 6950   INS01  : ISS;
 6951   LDST   : WR;
 6952 %}
 6953 
 6954 // Store - reg, mem
 6955 // Eg.  STR     x0, <mem>
 6956 pipe_class istore_reg_mem(iRegI src, memory mem)
 6957 %{
 6958   single_instruction;
 6959   mem    : ISS(read);
 6960   src    : EX2(read);
 6961   INS01  : ISS;
 6962   LDST   : WR;
 6963 %}
 6964 
 6965 // Store - reg, reg
 6966 // Eg. STR      x0, [sp, x1]
 6967 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6968 %{
 6969   single_instruction;
 6970   dst    : ISS(read);
 6971   src    : EX2(read);
 6972   INS01  : ISS;
 6973   LDST   : WR;
 6974 %}
 6975 
 6976 //------- Store pipeline operations -----------------------
 6977 
 6978 // Branch
 6979 pipe_class pipe_branch()
 6980 %{
 6981   single_instruction;
 6982   INS01  : ISS;
 6983   BRANCH : EX1;
 6984 %}
 6985 
 6986 // Conditional branch
 6987 pipe_class pipe_branch_cond(rFlagsReg cr)
 6988 %{
 6989   single_instruction;
 6990   cr     : EX1(read);
 6991   INS01  : ISS;
 6992   BRANCH : EX1;
 6993 %}
 6994 
 6995 // Compare & Branch
 6996 // EG.  CBZ/CBNZ
 6997 pipe_class pipe_cmp_branch(iRegI op1)
 6998 %{
 6999   single_instruction;
 7000   op1    : EX1(read);
 7001   INS01  : ISS;
 7002   BRANCH : EX1;
 7003 %}
 7004 
 7005 //------- Synchronisation operations ----------------------
 7006 
 7007 // Any operation requiring serialization.
 7008 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 7009 pipe_class pipe_serial()
 7010 %{
 7011   single_instruction;
 7012   force_serialization;
 7013   fixed_latency(16);
 7014   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7015   LDST   : WR;
 7016 %}
 7017 
 7018 // Generic big/slow expanded idiom - also serialized
 7019 pipe_class pipe_slow()
 7020 %{
 7021   instruction_count(10);
 7022   multiple_bundles;
 7023   force_serialization;
 7024   fixed_latency(16);
 7025   INS01  : ISS(2); // Cannot dual issue with any other instruction
 7026   LDST   : WR;
 7027 %}
 7028 
 7029 // Empty pipeline class
 7030 pipe_class pipe_class_empty()
 7031 %{
 7032   single_instruction;
 7033   fixed_latency(0);
 7034 %}
 7035 
 7036 // Default pipeline class.
 7037 pipe_class pipe_class_default()
 7038 %{
 7039   single_instruction;
 7040   fixed_latency(2);
 7041 %}
 7042 
 7043 // Pipeline class for compares.
 7044 pipe_class pipe_class_compare()
 7045 %{
 7046   single_instruction;
 7047   fixed_latency(16);
 7048 %}
 7049 
 7050 // Pipeline class for memory operations.
 7051 pipe_class pipe_class_memory()
 7052 %{
 7053   single_instruction;
 7054   fixed_latency(16);
 7055 %}
 7056 
 7057 // Pipeline class for call.
 7058 pipe_class pipe_class_call()
 7059 %{
 7060   single_instruction;
 7061   fixed_latency(100);
 7062 %}
 7063 
 7064 // Define the class for the Nop node.
 7065 define %{
 7066    MachNop = pipe_class_empty;
 7067 %}
 7068 
 7069 %}
 7070 //----------INSTRUCTIONS-------------------------------------------------------
 7071 //
 7072 // match      -- States which machine-independent subtree may be replaced
 7073 //               by this instruction.
 7074 // ins_cost   -- The estimated cost of this instruction is used by instruction
 7075 //               selection to identify a minimum cost tree of machine
 7076 //               instructions that matches a tree of machine-independent
 7077 //               instructions.
 7078 // format     -- A string providing the disassembly for this instruction.
 7079 //               The value of an instruction's operand may be inserted
 7080 //               by referring to it with a '$' prefix.
 7081 // opcode     -- Three instruction opcodes may be provided.  These are referred
 7082 //               to within an encode class as $primary, $secondary, and $tertiary
 7083 //               rrspectively.  The primary opcode is commonly used to
 7084 //               indicate the type of machine instruction, while secondary
 7085 //               and tertiary are often used for prefix options or addressing
 7086 //               modes.
 7087 // ins_encode -- A list of encode classes with parameters. The encode class
 7088 //               name must have been defined in an 'enc_class' specification
 7089 //               in the encode section of the architecture description.
 7090 
 7091 // ============================================================================
 7092 // Memory (Load/Store) Instructions
 7093 
 7094 // Load Instructions
 7095 
 7096 // Load Byte (8 bit signed)
 7097 instruct loadB(iRegINoSp dst, memory1 mem)
 7098 %{
 7099   match(Set dst (LoadB mem));
 7100   predicate(!needs_acquiring_load(n));
 7101 
 7102   ins_cost(4 * INSN_COST);
 7103   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7104 
 7105   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7106 
 7107   ins_pipe(iload_reg_mem);
 7108 %}
 7109 
 7110 // Load Byte (8 bit signed) into long
 7111 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7112 %{
 7113   match(Set dst (ConvI2L (LoadB mem)));
 7114   predicate(!needs_acquiring_load(n->in(1)));
 7115 
 7116   ins_cost(4 * INSN_COST);
 7117   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7118 
 7119   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7120 
 7121   ins_pipe(iload_reg_mem);
 7122 %}
 7123 
 7124 // Load Byte (8 bit unsigned)
 7125 instruct loadUB(iRegINoSp dst, memory1 mem)
 7126 %{
 7127   match(Set dst (LoadUB mem));
 7128   predicate(!needs_acquiring_load(n));
 7129 
 7130   ins_cost(4 * INSN_COST);
 7131   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7132 
 7133   ins_encode(aarch64_enc_ldrb(dst, mem));
 7134 
 7135   ins_pipe(iload_reg_mem);
 7136 %}
 7137 
 7138 // Load Byte (8 bit unsigned) into long
 7139 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7140 %{
 7141   match(Set dst (ConvI2L (LoadUB mem)));
 7142   predicate(!needs_acquiring_load(n->in(1)));
 7143 
 7144   ins_cost(4 * INSN_COST);
 7145   format %{ "ldrb  $dst, $mem\t# byte" %}
 7146 
 7147   ins_encode(aarch64_enc_ldrb(dst, mem));
 7148 
 7149   ins_pipe(iload_reg_mem);
 7150 %}
 7151 
 7152 // Load Short (16 bit signed)
 7153 instruct loadS(iRegINoSp dst, memory2 mem)
 7154 %{
 7155   match(Set dst (LoadS mem));
 7156   predicate(!needs_acquiring_load(n));
 7157 
 7158   ins_cost(4 * INSN_COST);
 7159   format %{ "ldrshw  $dst, $mem\t# short" %}
 7160 
 7161   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7162 
 7163   ins_pipe(iload_reg_mem);
 7164 %}
 7165 
 7166 // Load Short (16 bit signed) into long
 7167 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7168 %{
 7169   match(Set dst (ConvI2L (LoadS mem)));
 7170   predicate(!needs_acquiring_load(n->in(1)));
 7171 
 7172   ins_cost(4 * INSN_COST);
 7173   format %{ "ldrsh  $dst, $mem\t# short" %}
 7174 
 7175   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7176 
 7177   ins_pipe(iload_reg_mem);
 7178 %}
 7179 
 7180 // Load Char (16 bit unsigned)
 7181 instruct loadUS(iRegINoSp dst, memory2 mem)
 7182 %{
 7183   match(Set dst (LoadUS mem));
 7184   predicate(!needs_acquiring_load(n));
 7185 
 7186   ins_cost(4 * INSN_COST);
 7187   format %{ "ldrh  $dst, $mem\t# short" %}
 7188 
 7189   ins_encode(aarch64_enc_ldrh(dst, mem));
 7190 
 7191   ins_pipe(iload_reg_mem);
 7192 %}
 7193 
 7194 // Load Short/Char (16 bit unsigned) into long
 7195 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7196 %{
 7197   match(Set dst (ConvI2L (LoadUS mem)));
 7198   predicate(!needs_acquiring_load(n->in(1)));
 7199 
 7200   ins_cost(4 * INSN_COST);
 7201   format %{ "ldrh  $dst, $mem\t# short" %}
 7202 
 7203   ins_encode(aarch64_enc_ldrh(dst, mem));
 7204 
 7205   ins_pipe(iload_reg_mem);
 7206 %}
 7207 
 7208 // Load Integer (32 bit signed)
 7209 instruct loadI(iRegINoSp dst, memory4 mem)
 7210 %{
 7211   match(Set dst (LoadI mem));
 7212   predicate(!needs_acquiring_load(n));
 7213 
 7214   ins_cost(4 * INSN_COST);
 7215   format %{ "ldrw  $dst, $mem\t# int" %}
 7216 
 7217   ins_encode(aarch64_enc_ldrw(dst, mem));
 7218 
 7219   ins_pipe(iload_reg_mem);
 7220 %}
 7221 
 7222 // Load Integer (32 bit signed) into long
 7223 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7224 %{
 7225   match(Set dst (ConvI2L (LoadI mem)));
 7226   predicate(!needs_acquiring_load(n->in(1)));
 7227 
 7228   ins_cost(4 * INSN_COST);
 7229   format %{ "ldrsw  $dst, $mem\t# int" %}
 7230 
 7231   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7232 
 7233   ins_pipe(iload_reg_mem);
 7234 %}
 7235 
 7236 // Load Integer (32 bit unsigned) into long
 7237 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7238 %{
 7239   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7240   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7241 
 7242   ins_cost(4 * INSN_COST);
 7243   format %{ "ldrw  $dst, $mem\t# int" %}
 7244 
 7245   ins_encode(aarch64_enc_ldrw(dst, mem));
 7246 
 7247   ins_pipe(iload_reg_mem);
 7248 %}
 7249 
 7250 // Load Long (64 bit signed)
 7251 instruct loadL(iRegLNoSp dst, memory8 mem)
 7252 %{
 7253   match(Set dst (LoadL mem));
 7254   predicate(!needs_acquiring_load(n));
 7255 
 7256   ins_cost(4 * INSN_COST);
 7257   format %{ "ldr  $dst, $mem\t# int" %}
 7258 
 7259   ins_encode(aarch64_enc_ldr(dst, mem));
 7260 
 7261   ins_pipe(iload_reg_mem);
 7262 %}
 7263 
 7264 // Load Range
 7265 instruct loadRange(iRegINoSp dst, memory4 mem)
 7266 %{
 7267   match(Set dst (LoadRange mem));
 7268 
 7269   ins_cost(4 * INSN_COST);
 7270   format %{ "ldrw  $dst, $mem\t# range" %}
 7271 
 7272   ins_encode(aarch64_enc_ldrw(dst, mem));
 7273 
 7274   ins_pipe(iload_reg_mem);
 7275 %}
 7276 
 7277 // Load Pointer
 7278 instruct loadP(iRegPNoSp dst, memory8 mem)
 7279 %{
 7280   match(Set dst (LoadP mem));
 7281   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7282 
 7283   ins_cost(4 * INSN_COST);
 7284   format %{ "ldr  $dst, $mem\t# ptr" %}
 7285 
 7286   ins_encode(aarch64_enc_ldr(dst, mem));
 7287 
 7288   ins_pipe(iload_reg_mem);
 7289 %}
 7290 
 7291 // Load Compressed Pointer
 7292 instruct loadN(iRegNNoSp dst, memory4 mem)
 7293 %{
 7294   match(Set dst (LoadN mem));
 7295   predicate(!needs_acquiring_load(n));
 7296 
 7297   ins_cost(4 * INSN_COST);
 7298   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7299 
 7300   ins_encode(aarch64_enc_ldrw(dst, mem));
 7301 
 7302   ins_pipe(iload_reg_mem);
 7303 %}
 7304 
 7305 // Load Klass Pointer
 7306 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7307 %{
 7308   match(Set dst (LoadKlass mem));
 7309   predicate(!needs_acquiring_load(n));
 7310 
 7311   ins_cost(4 * INSN_COST);
 7312   format %{ "ldr  $dst, $mem\t# class" %}
 7313 
 7314   ins_encode(aarch64_enc_ldr(dst, mem));
 7315 
 7316   ins_pipe(iload_reg_mem);
 7317 %}
 7318 
 7319 // Load Narrow Klass Pointer
 7320 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7321 %{
 7322   match(Set dst (LoadNKlass mem));
 7323   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 7324 
 7325   ins_cost(4 * INSN_COST);
 7326   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7327 
 7328   ins_encode(aarch64_enc_ldrw(dst, mem));
 7329 
 7330   ins_pipe(iload_reg_mem);
 7331 %}
 7332 
 7333 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7334 %{
 7335   match(Set dst (LoadNKlass mem));
 7336   effect(KILL cr);
 7337   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 7338 
 7339   ins_cost(4 * INSN_COST);
 7340   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7341   ins_encode %{
 7342     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7343     assert($mem$$index$$Register == noreg, "expect no index");
 7344     __ load_nklass_compact($dst$$Register, $mem$$base$$Register);
 7345   %}
 7346   ins_pipe(pipe_slow);
 7347 %}
 7348 
 7349 // Load Float
 7350 instruct loadF(vRegF dst, memory4 mem)
 7351 %{
 7352   match(Set dst (LoadF mem));
 7353   predicate(!needs_acquiring_load(n));
 7354 
 7355   ins_cost(4 * INSN_COST);
 7356   format %{ "ldrs  $dst, $mem\t# float" %}
 7357 
 7358   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7359 
 7360   ins_pipe(pipe_class_memory);
 7361 %}
 7362 
 7363 // Load Double
 7364 instruct loadD(vRegD dst, memory8 mem)
 7365 %{
 7366   match(Set dst (LoadD mem));
 7367   predicate(!needs_acquiring_load(n));
 7368 
 7369   ins_cost(4 * INSN_COST);
 7370   format %{ "ldrd  $dst, $mem\t# double" %}
 7371 
 7372   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7373 
 7374   ins_pipe(pipe_class_memory);
 7375 %}
 7376 
 7377 
 7378 // Load Int Constant
 7379 instruct loadConI(iRegINoSp dst, immI src)
 7380 %{
 7381   match(Set dst src);
 7382 
 7383   ins_cost(INSN_COST);
 7384   format %{ "mov $dst, $src\t# int" %}
 7385 
 7386   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7387 
 7388   ins_pipe(ialu_imm);
 7389 %}
 7390 
 7391 // Load Long Constant
 7392 instruct loadConL(iRegLNoSp dst, immL src)
 7393 %{
 7394   match(Set dst src);
 7395 
 7396   ins_cost(INSN_COST);
 7397   format %{ "mov $dst, $src\t# long" %}
 7398 
 7399   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7400 
 7401   ins_pipe(ialu_imm);
 7402 %}
 7403 
 7404 // Load Pointer Constant
 7405 
 7406 instruct loadConP(iRegPNoSp dst, immP con)
 7407 %{
 7408   match(Set dst con);
 7409 
 7410   ins_cost(INSN_COST * 4);
 7411   format %{
 7412     "mov  $dst, $con\t# ptr\n\t"
 7413   %}
 7414 
 7415   ins_encode(aarch64_enc_mov_p(dst, con));
 7416 
 7417   ins_pipe(ialu_imm);
 7418 %}
 7419 
 7420 // Load Null Pointer Constant
 7421 
 7422 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7423 %{
 7424   match(Set dst con);
 7425 
 7426   ins_cost(INSN_COST);
 7427   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7428 
 7429   ins_encode(aarch64_enc_mov_p0(dst, con));
 7430 
 7431   ins_pipe(ialu_imm);
 7432 %}
 7433 
 7434 // Load Pointer Constant One
 7435 
 7436 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7437 %{
 7438   match(Set dst con);
 7439 
 7440   ins_cost(INSN_COST);
 7441   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7442 
 7443   ins_encode(aarch64_enc_mov_p1(dst, con));
 7444 
 7445   ins_pipe(ialu_imm);
 7446 %}
 7447 
 7448 // Load Byte Map Base Constant
 7449 
 7450 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7451 %{
 7452   match(Set dst con);
 7453 
 7454   ins_cost(INSN_COST);
 7455   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7456 
 7457   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7458 
 7459   ins_pipe(ialu_imm);
 7460 %}
 7461 
 7462 // Load Narrow Pointer Constant
 7463 
 7464 instruct loadConN(iRegNNoSp dst, immN con)
 7465 %{
 7466   match(Set dst con);
 7467 
 7468   ins_cost(INSN_COST * 4);
 7469   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7470 
 7471   ins_encode(aarch64_enc_mov_n(dst, con));
 7472 
 7473   ins_pipe(ialu_imm);
 7474 %}
 7475 
 7476 // Load Narrow Null Pointer Constant
 7477 
 7478 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7479 %{
 7480   match(Set dst con);
 7481 
 7482   ins_cost(INSN_COST);
 7483   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7484 
 7485   ins_encode(aarch64_enc_mov_n0(dst, con));
 7486 
 7487   ins_pipe(ialu_imm);
 7488 %}
 7489 
 7490 // Load Narrow Klass Constant
 7491 
 7492 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7493 %{
 7494   match(Set dst con);
 7495 
 7496   ins_cost(INSN_COST);
 7497   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7498 
 7499   ins_encode(aarch64_enc_mov_nk(dst, con));
 7500 
 7501   ins_pipe(ialu_imm);
 7502 %}
 7503 
 7504 // Load Packed Float Constant
 7505 
 7506 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7507   match(Set dst con);
 7508   ins_cost(INSN_COST * 4);
 7509   format %{ "fmovs  $dst, $con"%}
 7510   ins_encode %{
 7511     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7512   %}
 7513 
 7514   ins_pipe(fp_imm_s);
 7515 %}
 7516 
 7517 // Load Float Constant
 7518 
 7519 instruct loadConF(vRegF dst, immF con) %{
 7520   match(Set dst con);
 7521 
 7522   ins_cost(INSN_COST * 4);
 7523 
 7524   format %{
 7525     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7526   %}
 7527 
 7528   ins_encode %{
 7529     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7530   %}
 7531 
 7532   ins_pipe(fp_load_constant_s);
 7533 %}
 7534 
 7535 // Load Packed Double Constant
 7536 
 7537 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7538   match(Set dst con);
 7539   ins_cost(INSN_COST);
 7540   format %{ "fmovd  $dst, $con"%}
 7541   ins_encode %{
 7542     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7543   %}
 7544 
 7545   ins_pipe(fp_imm_d);
 7546 %}
 7547 
 7548 // Load Double Constant
 7549 
 7550 instruct loadConD(vRegD dst, immD con) %{
 7551   match(Set dst con);
 7552 
 7553   ins_cost(INSN_COST * 5);
 7554   format %{
 7555     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7556   %}
 7557 
 7558   ins_encode %{
 7559     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7560   %}
 7561 
 7562   ins_pipe(fp_load_constant_d);
 7563 %}
 7564 
 7565 // Store Instructions
 7566 
 7567 // Store CMS card-mark Immediate
 7568 instruct storeimmCM0(immI0 zero, memory1 mem)
 7569 %{
 7570   match(Set mem (StoreCM mem zero));
 7571 
 7572   ins_cost(INSN_COST);
 7573   format %{ "storestore (elided)\n\t"
 7574             "strb zr, $mem\t# byte" %}
 7575 
 7576   ins_encode(aarch64_enc_strb0(mem));
 7577 
 7578   ins_pipe(istore_mem);
 7579 %}
 7580 
 7581 // Store CMS card-mark Immediate with intervening StoreStore
 7582 // needed when using CMS with no conditional card marking
 7583 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7584 %{
 7585   match(Set mem (StoreCM mem zero));
 7586 
 7587   ins_cost(INSN_COST * 2);
 7588   format %{ "storestore\n\t"
 7589             "dmb ishst"
 7590             "\n\tstrb zr, $mem\t# byte" %}
 7591 
 7592   ins_encode(aarch64_enc_strb0_ordered(mem));
 7593 
 7594   ins_pipe(istore_mem);
 7595 %}
 7596 
 7597 // Store Byte
 7598 instruct storeB(iRegIorL2I src, memory1 mem)
 7599 %{
 7600   match(Set mem (StoreB mem src));
 7601   predicate(!needs_releasing_store(n));
 7602 
 7603   ins_cost(INSN_COST);
 7604   format %{ "strb  $src, $mem\t# byte" %}
 7605 
 7606   ins_encode(aarch64_enc_strb(src, mem));
 7607 
 7608   ins_pipe(istore_reg_mem);
 7609 %}
 7610 
 7611 
 7612 instruct storeimmB0(immI0 zero, memory1 mem)
 7613 %{
 7614   match(Set mem (StoreB mem zero));
 7615   predicate(!needs_releasing_store(n));
 7616 
 7617   ins_cost(INSN_COST);
 7618   format %{ "strb rscractch2, $mem\t# byte" %}
 7619 
 7620   ins_encode(aarch64_enc_strb0(mem));
 7621 
 7622   ins_pipe(istore_mem);
 7623 %}
 7624 
 7625 // Store Char/Short
 7626 instruct storeC(iRegIorL2I src, memory2 mem)
 7627 %{
 7628   match(Set mem (StoreC mem src));
 7629   predicate(!needs_releasing_store(n));
 7630 
 7631   ins_cost(INSN_COST);
 7632   format %{ "strh  $src, $mem\t# short" %}
 7633 
 7634   ins_encode(aarch64_enc_strh(src, mem));
 7635 
 7636   ins_pipe(istore_reg_mem);
 7637 %}
 7638 
 7639 instruct storeimmC0(immI0 zero, memory2 mem)
 7640 %{
 7641   match(Set mem (StoreC mem zero));
 7642   predicate(!needs_releasing_store(n));
 7643 
 7644   ins_cost(INSN_COST);
 7645   format %{ "strh  zr, $mem\t# short" %}
 7646 
 7647   ins_encode(aarch64_enc_strh0(mem));
 7648 
 7649   ins_pipe(istore_mem);
 7650 %}
 7651 
 7652 // Store Integer
 7653 
 7654 instruct storeI(iRegIorL2I src, memory4 mem)
 7655 %{
 7656   match(Set mem(StoreI mem src));
 7657   predicate(!needs_releasing_store(n));
 7658 
 7659   ins_cost(INSN_COST);
 7660   format %{ "strw  $src, $mem\t# int" %}
 7661 
 7662   ins_encode(aarch64_enc_strw(src, mem));
 7663 
 7664   ins_pipe(istore_reg_mem);
 7665 %}
 7666 
 7667 instruct storeimmI0(immI0 zero, memory4 mem)
 7668 %{
 7669   match(Set mem(StoreI mem zero));
 7670   predicate(!needs_releasing_store(n));
 7671 
 7672   ins_cost(INSN_COST);
 7673   format %{ "strw  zr, $mem\t# int" %}
 7674 
 7675   ins_encode(aarch64_enc_strw0(mem));
 7676 
 7677   ins_pipe(istore_mem);
 7678 %}
 7679 
 7680 // Store Long (64 bit signed)
 7681 instruct storeL(iRegL src, memory8 mem)
 7682 %{
 7683   match(Set mem (StoreL mem src));
 7684   predicate(!needs_releasing_store(n));
 7685 
 7686   ins_cost(INSN_COST);
 7687   format %{ "str  $src, $mem\t# int" %}
 7688 
 7689   ins_encode(aarch64_enc_str(src, mem));
 7690 
 7691   ins_pipe(istore_reg_mem);
 7692 %}
 7693 
 7694 // Store Long (64 bit signed)
 7695 instruct storeimmL0(immL0 zero, memory8 mem)
 7696 %{
 7697   match(Set mem (StoreL mem zero));
 7698   predicate(!needs_releasing_store(n));
 7699 
 7700   ins_cost(INSN_COST);
 7701   format %{ "str  zr, $mem\t# int" %}
 7702 
 7703   ins_encode(aarch64_enc_str0(mem));
 7704 
 7705   ins_pipe(istore_mem);
 7706 %}
 7707 
 7708 // Store Pointer
 7709 instruct storeP(iRegP src, memory8 mem)
 7710 %{
 7711   match(Set mem (StoreP mem src));
 7712   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7713 
 7714   ins_cost(INSN_COST);
 7715   format %{ "str  $src, $mem\t# ptr" %}
 7716 
 7717   ins_encode(aarch64_enc_str(src, mem));
 7718 
 7719   ins_pipe(istore_reg_mem);
 7720 %}
 7721 
 7722 // Store Pointer
 7723 instruct storeimmP0(immP0 zero, memory8 mem)
 7724 %{
 7725   match(Set mem (StoreP mem zero));
 7726   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7727 
 7728   ins_cost(INSN_COST);
 7729   format %{ "str zr, $mem\t# ptr" %}
 7730 
 7731   ins_encode(aarch64_enc_str0(mem));
 7732 
 7733   ins_pipe(istore_mem);
 7734 %}
 7735 
 7736 // Store Compressed Pointer
 7737 instruct storeN(iRegN src, memory4 mem)
 7738 %{
 7739   match(Set mem (StoreN mem src));
 7740   predicate(!needs_releasing_store(n));
 7741 
 7742   ins_cost(INSN_COST);
 7743   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7744 
 7745   ins_encode(aarch64_enc_strw(src, mem));
 7746 
 7747   ins_pipe(istore_reg_mem);
 7748 %}
 7749 
 7750 instruct storeImmN0(immN0 zero, memory4 mem)
 7751 %{
 7752   match(Set mem (StoreN mem zero));
 7753   predicate(!needs_releasing_store(n));
 7754 
 7755   ins_cost(INSN_COST);
 7756   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7757 
 7758   ins_encode(aarch64_enc_strw0(mem));
 7759 
 7760   ins_pipe(istore_mem);
 7761 %}
 7762 
 7763 // Store Float
 7764 instruct storeF(vRegF src, memory4 mem)
 7765 %{
 7766   match(Set mem (StoreF mem src));
 7767   predicate(!needs_releasing_store(n));
 7768 
 7769   ins_cost(INSN_COST);
 7770   format %{ "strs  $src, $mem\t# float" %}
 7771 
 7772   ins_encode( aarch64_enc_strs(src, mem) );
 7773 
 7774   ins_pipe(pipe_class_memory);
 7775 %}
 7776 
 7777 // TODO
 7778 // implement storeImmF0 and storeFImmPacked
 7779 
 7780 // Store Double
 7781 instruct storeD(vRegD src, memory8 mem)
 7782 %{
 7783   match(Set mem (StoreD mem src));
 7784   predicate(!needs_releasing_store(n));
 7785 
 7786   ins_cost(INSN_COST);
 7787   format %{ "strd  $src, $mem\t# double" %}
 7788 
 7789   ins_encode( aarch64_enc_strd(src, mem) );
 7790 
 7791   ins_pipe(pipe_class_memory);
 7792 %}
 7793 
 7794 // Store Compressed Klass Pointer
 7795 instruct storeNKlass(iRegN src, memory4 mem)
 7796 %{
 7797   predicate(!needs_releasing_store(n));
 7798   match(Set mem (StoreNKlass mem src));
 7799 
 7800   ins_cost(INSN_COST);
 7801   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7802 
 7803   ins_encode(aarch64_enc_strw(src, mem));
 7804 
 7805   ins_pipe(istore_reg_mem);
 7806 %}
 7807 
 7808 // TODO
 7809 // implement storeImmD0 and storeDImmPacked
 7810 
 7811 // prefetch instructions
 7812 // Must be safe to execute with invalid address (cannot fault).
 7813 
 7814 instruct prefetchalloc( memory8 mem ) %{
 7815   match(PrefetchAllocation mem);
 7816 
 7817   ins_cost(INSN_COST);
 7818   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7819 
 7820   ins_encode( aarch64_enc_prefetchw(mem) );
 7821 
 7822   ins_pipe(iload_prefetch);
 7823 %}
 7824 
 7825 //  ---------------- volatile loads and stores ----------------
 7826 
 7827 // Load Byte (8 bit signed)
 7828 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7829 %{
 7830   match(Set dst (LoadB mem));
 7831 
 7832   ins_cost(VOLATILE_REF_COST);
 7833   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7834 
 7835   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7836 
 7837   ins_pipe(pipe_serial);
 7838 %}
 7839 
 7840 // Load Byte (8 bit signed) into long
 7841 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7842 %{
 7843   match(Set dst (ConvI2L (LoadB mem)));
 7844 
 7845   ins_cost(VOLATILE_REF_COST);
 7846   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7847 
 7848   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7849 
 7850   ins_pipe(pipe_serial);
 7851 %}
 7852 
 7853 // Load Byte (8 bit unsigned)
 7854 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7855 %{
 7856   match(Set dst (LoadUB mem));
 7857 
 7858   ins_cost(VOLATILE_REF_COST);
 7859   format %{ "ldarb  $dst, $mem\t# byte" %}
 7860 
 7861   ins_encode(aarch64_enc_ldarb(dst, mem));
 7862 
 7863   ins_pipe(pipe_serial);
 7864 %}
 7865 
 7866 // Load Byte (8 bit unsigned) into long
 7867 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7868 %{
 7869   match(Set dst (ConvI2L (LoadUB mem)));
 7870 
 7871   ins_cost(VOLATILE_REF_COST);
 7872   format %{ "ldarb  $dst, $mem\t# byte" %}
 7873 
 7874   ins_encode(aarch64_enc_ldarb(dst, mem));
 7875 
 7876   ins_pipe(pipe_serial);
 7877 %}
 7878 
 7879 // Load Short (16 bit signed)
 7880 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7881 %{
 7882   match(Set dst (LoadS mem));
 7883 
 7884   ins_cost(VOLATILE_REF_COST);
 7885   format %{ "ldarshw  $dst, $mem\t# short" %}
 7886 
 7887   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7888 
 7889   ins_pipe(pipe_serial);
 7890 %}
 7891 
 7892 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7893 %{
 7894   match(Set dst (LoadUS mem));
 7895 
 7896   ins_cost(VOLATILE_REF_COST);
 7897   format %{ "ldarhw  $dst, $mem\t# short" %}
 7898 
 7899   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7900 
 7901   ins_pipe(pipe_serial);
 7902 %}
 7903 
 7904 // Load Short/Char (16 bit unsigned) into long
 7905 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7906 %{
 7907   match(Set dst (ConvI2L (LoadUS mem)));
 7908 
 7909   ins_cost(VOLATILE_REF_COST);
 7910   format %{ "ldarh  $dst, $mem\t# short" %}
 7911 
 7912   ins_encode(aarch64_enc_ldarh(dst, mem));
 7913 
 7914   ins_pipe(pipe_serial);
 7915 %}
 7916 
 7917 // Load Short/Char (16 bit signed) into long
 7918 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7919 %{
 7920   match(Set dst (ConvI2L (LoadS mem)));
 7921 
 7922   ins_cost(VOLATILE_REF_COST);
 7923   format %{ "ldarh  $dst, $mem\t# short" %}
 7924 
 7925   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7926 
 7927   ins_pipe(pipe_serial);
 7928 %}
 7929 
 7930 // Load Integer (32 bit signed)
 7931 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7932 %{
 7933   match(Set dst (LoadI mem));
 7934 
 7935   ins_cost(VOLATILE_REF_COST);
 7936   format %{ "ldarw  $dst, $mem\t# int" %}
 7937 
 7938   ins_encode(aarch64_enc_ldarw(dst, mem));
 7939 
 7940   ins_pipe(pipe_serial);
 7941 %}
 7942 
 7943 // Load Integer (32 bit unsigned) into long
 7944 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7945 %{
 7946   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7947 
 7948   ins_cost(VOLATILE_REF_COST);
 7949   format %{ "ldarw  $dst, $mem\t# int" %}
 7950 
 7951   ins_encode(aarch64_enc_ldarw(dst, mem));
 7952 
 7953   ins_pipe(pipe_serial);
 7954 %}
 7955 
 7956 // Load Long (64 bit signed)
 7957 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7958 %{
 7959   match(Set dst (LoadL mem));
 7960 
 7961   ins_cost(VOLATILE_REF_COST);
 7962   format %{ "ldar  $dst, $mem\t# int" %}
 7963 
 7964   ins_encode(aarch64_enc_ldar(dst, mem));
 7965 
 7966   ins_pipe(pipe_serial);
 7967 %}
 7968 
 7969 // Load Pointer
 7970 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7971 %{
 7972   match(Set dst (LoadP mem));
 7973   predicate(n->as_Load()->barrier_data() == 0);
 7974 
 7975   ins_cost(VOLATILE_REF_COST);
 7976   format %{ "ldar  $dst, $mem\t# ptr" %}
 7977 
 7978   ins_encode(aarch64_enc_ldar(dst, mem));
 7979 
 7980   ins_pipe(pipe_serial);
 7981 %}
 7982 
 7983 // Load Compressed Pointer
 7984 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7985 %{
 7986   match(Set dst (LoadN mem));
 7987 
 7988   ins_cost(VOLATILE_REF_COST);
 7989   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7990 
 7991   ins_encode(aarch64_enc_ldarw(dst, mem));
 7992 
 7993   ins_pipe(pipe_serial);
 7994 %}
 7995 
 7996 // Load Float
 7997 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7998 %{
 7999   match(Set dst (LoadF mem));
 8000 
 8001   ins_cost(VOLATILE_REF_COST);
 8002   format %{ "ldars  $dst, $mem\t# float" %}
 8003 
 8004   ins_encode( aarch64_enc_fldars(dst, mem) );
 8005 
 8006   ins_pipe(pipe_serial);
 8007 %}
 8008 
 8009 // Load Double
 8010 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 8011 %{
 8012   match(Set dst (LoadD mem));
 8013 
 8014   ins_cost(VOLATILE_REF_COST);
 8015   format %{ "ldard  $dst, $mem\t# double" %}
 8016 
 8017   ins_encode( aarch64_enc_fldard(dst, mem) );
 8018 
 8019   ins_pipe(pipe_serial);
 8020 %}
 8021 
 8022 // Store Byte
 8023 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8024 %{
 8025   match(Set mem (StoreB mem src));
 8026 
 8027   ins_cost(VOLATILE_REF_COST);
 8028   format %{ "stlrb  $src, $mem\t# byte" %}
 8029 
 8030   ins_encode(aarch64_enc_stlrb(src, mem));
 8031 
 8032   ins_pipe(pipe_class_memory);
 8033 %}
 8034 
 8035 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8036 %{
 8037   match(Set mem (StoreB mem zero));
 8038 
 8039   ins_cost(VOLATILE_REF_COST);
 8040   format %{ "stlrb  zr, $mem\t# byte" %}
 8041 
 8042   ins_encode(aarch64_enc_stlrb0(mem));
 8043 
 8044   ins_pipe(pipe_class_memory);
 8045 %}
 8046 
 8047 // Store Char/Short
 8048 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8049 %{
 8050   match(Set mem (StoreC mem src));
 8051 
 8052   ins_cost(VOLATILE_REF_COST);
 8053   format %{ "stlrh  $src, $mem\t# short" %}
 8054 
 8055   ins_encode(aarch64_enc_stlrh(src, mem));
 8056 
 8057   ins_pipe(pipe_class_memory);
 8058 %}
 8059 
 8060 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8061 %{
 8062   match(Set mem (StoreC mem zero));
 8063 
 8064   ins_cost(VOLATILE_REF_COST);
 8065   format %{ "stlrh  zr, $mem\t# short" %}
 8066 
 8067   ins_encode(aarch64_enc_stlrh0(mem));
 8068 
 8069   ins_pipe(pipe_class_memory);
 8070 %}
 8071 
 8072 // Store Integer
 8073 
 8074 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 8075 %{
 8076   match(Set mem(StoreI mem src));
 8077 
 8078   ins_cost(VOLATILE_REF_COST);
 8079   format %{ "stlrw  $src, $mem\t# int" %}
 8080 
 8081   ins_encode(aarch64_enc_stlrw(src, mem));
 8082 
 8083   ins_pipe(pipe_class_memory);
 8084 %}
 8085 
 8086 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 8087 %{
 8088   match(Set mem(StoreI mem zero));
 8089 
 8090   ins_cost(VOLATILE_REF_COST);
 8091   format %{ "stlrw  zr, $mem\t# int" %}
 8092 
 8093   ins_encode(aarch64_enc_stlrw0(mem));
 8094 
 8095   ins_pipe(pipe_class_memory);
 8096 %}
 8097 
 8098 // Store Long (64 bit signed)
 8099 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8100 %{
 8101   match(Set mem (StoreL mem src));
 8102 
 8103   ins_cost(VOLATILE_REF_COST);
 8104   format %{ "stlr  $src, $mem\t# int" %}
 8105 
 8106   ins_encode(aarch64_enc_stlr(src, mem));
 8107 
 8108   ins_pipe(pipe_class_memory);
 8109 %}
 8110 
 8111 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8112 %{
 8113   match(Set mem (StoreL mem zero));
 8114 
 8115   ins_cost(VOLATILE_REF_COST);
 8116   format %{ "stlr  zr, $mem\t# int" %}
 8117 
 8118   ins_encode(aarch64_enc_stlr0(mem));
 8119 
 8120   ins_pipe(pipe_class_memory);
 8121 %}
 8122 
 8123 // Store Pointer
 8124 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8125 %{
 8126   match(Set mem (StoreP mem src));
 8127   predicate(n->as_Store()->barrier_data() == 0);
 8128 
 8129   ins_cost(VOLATILE_REF_COST);
 8130   format %{ "stlr  $src, $mem\t# ptr" %}
 8131 
 8132   ins_encode(aarch64_enc_stlr(src, mem));
 8133 
 8134   ins_pipe(pipe_class_memory);
 8135 %}
 8136 
 8137 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8138 %{
 8139   match(Set mem (StoreP mem zero));
 8140   predicate(n->as_Store()->barrier_data() == 0);
 8141 
 8142   ins_cost(VOLATILE_REF_COST);
 8143   format %{ "stlr  zr, $mem\t# ptr" %}
 8144 
 8145   ins_encode(aarch64_enc_stlr0(mem));
 8146 
 8147   ins_pipe(pipe_class_memory);
 8148 %}
 8149 
 8150 // Store Compressed Pointer
 8151 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8152 %{
 8153   match(Set mem (StoreN mem src));
 8154 
 8155   ins_cost(VOLATILE_REF_COST);
 8156   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8157 
 8158   ins_encode(aarch64_enc_stlrw(src, mem));
 8159 
 8160   ins_pipe(pipe_class_memory);
 8161 %}
 8162 
 8163 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8164 %{
 8165   match(Set mem (StoreN mem zero));
 8166 
 8167   ins_cost(VOLATILE_REF_COST);
 8168   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8169 
 8170   ins_encode(aarch64_enc_stlrw0(mem));
 8171 
 8172   ins_pipe(pipe_class_memory);
 8173 %}
 8174 
 8175 // Store Float
 8176 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8177 %{
 8178   match(Set mem (StoreF mem src));
 8179 
 8180   ins_cost(VOLATILE_REF_COST);
 8181   format %{ "stlrs  $src, $mem\t# float" %}
 8182 
 8183   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8184 
 8185   ins_pipe(pipe_class_memory);
 8186 %}
 8187 
 8188 // TODO
 8189 // implement storeImmF0 and storeFImmPacked
 8190 
 8191 // Store Double
 8192 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8193 %{
 8194   match(Set mem (StoreD mem src));
 8195 
 8196   ins_cost(VOLATILE_REF_COST);
 8197   format %{ "stlrd  $src, $mem\t# double" %}
 8198 
 8199   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8200 
 8201   ins_pipe(pipe_class_memory);
 8202 %}
 8203 
 8204 //  ---------------- end of volatile loads and stores ----------------
 8205 
 8206 instruct cacheWB(indirect addr)
 8207 %{
 8208   predicate(VM_Version::supports_data_cache_line_flush());
 8209   match(CacheWB addr);
 8210 
 8211   ins_cost(100);
 8212   format %{"cache wb $addr" %}
 8213   ins_encode %{
 8214     assert($addr->index_position() < 0, "should be");
 8215     assert($addr$$disp == 0, "should be");
 8216     __ cache_wb(Address($addr$$base$$Register, 0));
 8217   %}
 8218   ins_pipe(pipe_slow); // XXX
 8219 %}
 8220 
 8221 instruct cacheWBPreSync()
 8222 %{
 8223   predicate(VM_Version::supports_data_cache_line_flush());
 8224   match(CacheWBPreSync);
 8225 
 8226   ins_cost(100);
 8227   format %{"cache wb presync" %}
 8228   ins_encode %{
 8229     __ cache_wbsync(true);
 8230   %}
 8231   ins_pipe(pipe_slow); // XXX
 8232 %}
 8233 
 8234 instruct cacheWBPostSync()
 8235 %{
 8236   predicate(VM_Version::supports_data_cache_line_flush());
 8237   match(CacheWBPostSync);
 8238 
 8239   ins_cost(100);
 8240   format %{"cache wb postsync" %}
 8241   ins_encode %{
 8242     __ cache_wbsync(false);
 8243   %}
 8244   ins_pipe(pipe_slow); // XXX
 8245 %}
 8246 
 8247 // ============================================================================
 8248 // BSWAP Instructions
 8249 
 8250 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8251   match(Set dst (ReverseBytesI src));
 8252 
 8253   ins_cost(INSN_COST);
 8254   format %{ "revw  $dst, $src" %}
 8255 
 8256   ins_encode %{
 8257     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8258   %}
 8259 
 8260   ins_pipe(ialu_reg);
 8261 %}
 8262 
 8263 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8264   match(Set dst (ReverseBytesL src));
 8265 
 8266   ins_cost(INSN_COST);
 8267   format %{ "rev  $dst, $src" %}
 8268 
 8269   ins_encode %{
 8270     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8271   %}
 8272 
 8273   ins_pipe(ialu_reg);
 8274 %}
 8275 
 8276 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8277   match(Set dst (ReverseBytesUS src));
 8278 
 8279   ins_cost(INSN_COST);
 8280   format %{ "rev16w  $dst, $src" %}
 8281 
 8282   ins_encode %{
 8283     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8284   %}
 8285 
 8286   ins_pipe(ialu_reg);
 8287 %}
 8288 
 8289 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8290   match(Set dst (ReverseBytesS src));
 8291 
 8292   ins_cost(INSN_COST);
 8293   format %{ "rev16w  $dst, $src\n\t"
 8294             "sbfmw $dst, $dst, #0, #15" %}
 8295 
 8296   ins_encode %{
 8297     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8298     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8299   %}
 8300 
 8301   ins_pipe(ialu_reg);
 8302 %}
 8303 
 8304 // ============================================================================
 8305 // Zero Count Instructions
 8306 
 8307 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8308   match(Set dst (CountLeadingZerosI src));
 8309 
 8310   ins_cost(INSN_COST);
 8311   format %{ "clzw  $dst, $src" %}
 8312   ins_encode %{
 8313     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8314   %}
 8315 
 8316   ins_pipe(ialu_reg);
 8317 %}
 8318 
 8319 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8320   match(Set dst (CountLeadingZerosL src));
 8321 
 8322   ins_cost(INSN_COST);
 8323   format %{ "clz   $dst, $src" %}
 8324   ins_encode %{
 8325     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8326   %}
 8327 
 8328   ins_pipe(ialu_reg);
 8329 %}
 8330 
 8331 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8332   match(Set dst (CountTrailingZerosI src));
 8333 
 8334   ins_cost(INSN_COST * 2);
 8335   format %{ "rbitw  $dst, $src\n\t"
 8336             "clzw   $dst, $dst" %}
 8337   ins_encode %{
 8338     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8339     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8340   %}
 8341 
 8342   ins_pipe(ialu_reg);
 8343 %}
 8344 
 8345 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8346   match(Set dst (CountTrailingZerosL src));
 8347 
 8348   ins_cost(INSN_COST * 2);
 8349   format %{ "rbit   $dst, $src\n\t"
 8350             "clz    $dst, $dst" %}
 8351   ins_encode %{
 8352     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8353     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8354   %}
 8355 
 8356   ins_pipe(ialu_reg);
 8357 %}
 8358 
 8359 //---------- Population Count Instructions -------------------------------------
 8360 //
 8361 
 8362 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8363   match(Set dst (PopCountI src));
 8364   effect(TEMP tmp);
 8365   ins_cost(INSN_COST * 13);
 8366 
 8367   format %{ "movw   $src, $src\n\t"
 8368             "mov    $tmp, $src\t# vector (1D)\n\t"
 8369             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8370             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8371             "mov    $dst, $tmp\t# vector (1D)" %}
 8372   ins_encode %{
 8373     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8374     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8375     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8376     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8377     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8378   %}
 8379 
 8380   ins_pipe(pipe_class_default);
 8381 %}
 8382 
 8383 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8384   match(Set dst (PopCountI (LoadI mem)));
 8385   effect(TEMP tmp);
 8386   ins_cost(INSN_COST * 13);
 8387 
 8388   format %{ "ldrs   $tmp, $mem\n\t"
 8389             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8390             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8391             "mov    $dst, $tmp\t# vector (1D)" %}
 8392   ins_encode %{
 8393     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8394     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8395               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8396     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8397     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8398     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8399   %}
 8400 
 8401   ins_pipe(pipe_class_default);
 8402 %}
 8403 
 8404 // Note: Long.bitCount(long) returns an int.
 8405 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8406   match(Set dst (PopCountL src));
 8407   effect(TEMP tmp);
 8408   ins_cost(INSN_COST * 13);
 8409 
 8410   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8411             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8412             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8413             "mov    $dst, $tmp\t# vector (1D)" %}
 8414   ins_encode %{
 8415     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8416     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8417     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8418     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8419   %}
 8420 
 8421   ins_pipe(pipe_class_default);
 8422 %}
 8423 
 8424 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8425   match(Set dst (PopCountL (LoadL mem)));
 8426   effect(TEMP tmp);
 8427   ins_cost(INSN_COST * 13);
 8428 
 8429   format %{ "ldrd   $tmp, $mem\n\t"
 8430             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8431             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8432             "mov    $dst, $tmp\t# vector (1D)" %}
 8433   ins_encode %{
 8434     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8435     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8436               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8437     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8438     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8439     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8440   %}
 8441 
 8442   ins_pipe(pipe_class_default);
 8443 %}
 8444 
 8445 // ============================================================================
 8446 // MemBar Instruction
 8447 
 8448 instruct load_fence() %{
 8449   match(LoadFence);
 8450   ins_cost(VOLATILE_REF_COST);
 8451 
 8452   format %{ "load_fence" %}
 8453 
 8454   ins_encode %{
 8455     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8456   %}
 8457   ins_pipe(pipe_serial);
 8458 %}
 8459 
 8460 instruct unnecessary_membar_acquire() %{
 8461   predicate(unnecessary_acquire(n));
 8462   match(MemBarAcquire);
 8463   ins_cost(0);
 8464 
 8465   format %{ "membar_acquire (elided)" %}
 8466 
 8467   ins_encode %{
 8468     __ block_comment("membar_acquire (elided)");
 8469   %}
 8470 
 8471   ins_pipe(pipe_class_empty);
 8472 %}
 8473 
 8474 instruct membar_acquire() %{
 8475   match(MemBarAcquire);
 8476   ins_cost(VOLATILE_REF_COST);
 8477 
 8478   format %{ "membar_acquire\n\t"
 8479             "dmb ish" %}
 8480 
 8481   ins_encode %{
 8482     __ block_comment("membar_acquire");
 8483     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8484   %}
 8485 
 8486   ins_pipe(pipe_serial);
 8487 %}
 8488 
 8489 
 8490 instruct membar_acquire_lock() %{
 8491   match(MemBarAcquireLock);
 8492   ins_cost(VOLATILE_REF_COST);
 8493 
 8494   format %{ "membar_acquire_lock (elided)" %}
 8495 
 8496   ins_encode %{
 8497     __ block_comment("membar_acquire_lock (elided)");
 8498   %}
 8499 
 8500   ins_pipe(pipe_serial);
 8501 %}
 8502 
 8503 instruct store_fence() %{
 8504   match(StoreFence);
 8505   ins_cost(VOLATILE_REF_COST);
 8506 
 8507   format %{ "store_fence" %}
 8508 
 8509   ins_encode %{
 8510     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8511   %}
 8512   ins_pipe(pipe_serial);
 8513 %}
 8514 
 8515 instruct unnecessary_membar_release() %{
 8516   predicate(unnecessary_release(n));
 8517   match(MemBarRelease);
 8518   ins_cost(0);
 8519 
 8520   format %{ "membar_release (elided)" %}
 8521 
 8522   ins_encode %{
 8523     __ block_comment("membar_release (elided)");
 8524   %}
 8525   ins_pipe(pipe_serial);
 8526 %}
 8527 
 8528 instruct membar_release() %{
 8529   match(MemBarRelease);
 8530   ins_cost(VOLATILE_REF_COST);
 8531 
 8532   format %{ "membar_release\n\t"
 8533             "dmb ish" %}
 8534 
 8535   ins_encode %{
 8536     __ block_comment("membar_release");
 8537     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8538   %}
 8539   ins_pipe(pipe_serial);
 8540 %}
 8541 
 8542 instruct membar_storestore() %{
 8543   match(MemBarStoreStore);
 8544   match(StoreStoreFence);
 8545   ins_cost(VOLATILE_REF_COST);
 8546 
 8547   format %{ "MEMBAR-store-store" %}
 8548 
 8549   ins_encode %{
 8550     __ membar(Assembler::StoreStore);
 8551   %}
 8552   ins_pipe(pipe_serial);
 8553 %}
 8554 
 8555 instruct membar_release_lock() %{
 8556   match(MemBarReleaseLock);
 8557   ins_cost(VOLATILE_REF_COST);
 8558 
 8559   format %{ "membar_release_lock (elided)" %}
 8560 
 8561   ins_encode %{
 8562     __ block_comment("membar_release_lock (elided)");
 8563   %}
 8564 
 8565   ins_pipe(pipe_serial);
 8566 %}
 8567 
 8568 instruct unnecessary_membar_volatile() %{
 8569   predicate(unnecessary_volatile(n));
 8570   match(MemBarVolatile);
 8571   ins_cost(0);
 8572 
 8573   format %{ "membar_volatile (elided)" %}
 8574 
 8575   ins_encode %{
 8576     __ block_comment("membar_volatile (elided)");
 8577   %}
 8578 
 8579   ins_pipe(pipe_serial);
 8580 %}
 8581 
 8582 instruct membar_volatile() %{
 8583   match(MemBarVolatile);
 8584   ins_cost(VOLATILE_REF_COST*100);
 8585 
 8586   format %{ "membar_volatile\n\t"
 8587              "dmb ish"%}
 8588 
 8589   ins_encode %{
 8590     __ block_comment("membar_volatile");
 8591     __ membar(Assembler::StoreLoad);
 8592   %}
 8593 
 8594   ins_pipe(pipe_serial);
 8595 %}
 8596 
 8597 // ============================================================================
 8598 // Cast/Convert Instructions
 8599 
 8600 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8601   match(Set dst (CastX2P src));
 8602 
 8603   ins_cost(INSN_COST);
 8604   format %{ "mov $dst, $src\t# long -> ptr" %}
 8605 
 8606   ins_encode %{
 8607     if ($dst$$reg != $src$$reg) {
 8608       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8609     }
 8610   %}
 8611 
 8612   ins_pipe(ialu_reg);
 8613 %}
 8614 
 8615 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8616   match(Set dst (CastP2X src));
 8617 
 8618   ins_cost(INSN_COST);
 8619   format %{ "mov $dst, $src\t# ptr -> long" %}
 8620 
 8621   ins_encode %{
 8622     if ($dst$$reg != $src$$reg) {
 8623       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8624     }
 8625   %}
 8626 
 8627   ins_pipe(ialu_reg);
 8628 %}
 8629 
 8630 // Convert oop into int for vectors alignment masking
 8631 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8632   match(Set dst (ConvL2I (CastP2X src)));
 8633 
 8634   ins_cost(INSN_COST);
 8635   format %{ "movw $dst, $src\t# ptr -> int" %}
 8636   ins_encode %{
 8637     __ movw($dst$$Register, $src$$Register);
 8638   %}
 8639 
 8640   ins_pipe(ialu_reg);
 8641 %}
 8642 
 8643 // Convert compressed oop into int for vectors alignment masking
 8644 // in case of 32bit oops (heap < 4Gb).
 8645 instruct convN2I(iRegINoSp dst, iRegN src)
 8646 %{
 8647   predicate(CompressedOops::shift() == 0);
 8648   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8649 
 8650   ins_cost(INSN_COST);
 8651   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8652   ins_encode %{
 8653     __ movw($dst$$Register, $src$$Register);
 8654   %}
 8655 
 8656   ins_pipe(ialu_reg);
 8657 %}
 8658 
 8659 
 8660 // Convert oop pointer into compressed form
 8661 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8662   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8663   match(Set dst (EncodeP src));
 8664   effect(KILL cr);
 8665   ins_cost(INSN_COST * 3);
 8666   format %{ "encode_heap_oop $dst, $src" %}
 8667   ins_encode %{
 8668     Register s = $src$$Register;
 8669     Register d = $dst$$Register;
 8670     __ encode_heap_oop(d, s);
 8671   %}
 8672   ins_pipe(ialu_reg);
 8673 %}
 8674 
 8675 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8676   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8677   match(Set dst (EncodeP src));
 8678   ins_cost(INSN_COST * 3);
 8679   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8680   ins_encode %{
 8681     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8682   %}
 8683   ins_pipe(ialu_reg);
 8684 %}
 8685 
 8686 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8687   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8688             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8689   match(Set dst (DecodeN src));
 8690   ins_cost(INSN_COST * 3);
 8691   format %{ "decode_heap_oop $dst, $src" %}
 8692   ins_encode %{
 8693     Register s = $src$$Register;
 8694     Register d = $dst$$Register;
 8695     __ decode_heap_oop(d, s);
 8696   %}
 8697   ins_pipe(ialu_reg);
 8698 %}
 8699 
 8700 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8701   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8702             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8703   match(Set dst (DecodeN src));
 8704   ins_cost(INSN_COST * 3);
 8705   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8706   ins_encode %{
 8707     Register s = $src$$Register;
 8708     Register d = $dst$$Register;
 8709     __ decode_heap_oop_not_null(d, s);
 8710   %}
 8711   ins_pipe(ialu_reg);
 8712 %}
 8713 
 8714 // n.b. AArch64 implementations of encode_klass_not_null and
 8715 // decode_klass_not_null do not modify the flags register so, unlike
 8716 // Intel, we don't kill CR as a side effect here
 8717 
 8718 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8719   match(Set dst (EncodePKlass src));
 8720 
 8721   ins_cost(INSN_COST * 3);
 8722   format %{ "encode_klass_not_null $dst,$src" %}
 8723 
 8724   ins_encode %{
 8725     Register src_reg = as_Register($src$$reg);
 8726     Register dst_reg = as_Register($dst$$reg);
 8727     __ encode_klass_not_null(dst_reg, src_reg);
 8728   %}
 8729 
 8730    ins_pipe(ialu_reg);
 8731 %}
 8732 
 8733 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8734   match(Set dst (DecodeNKlass src));
 8735 
 8736   ins_cost(INSN_COST * 3);
 8737   format %{ "decode_klass_not_null $dst,$src" %}
 8738 
 8739   ins_encode %{
 8740     Register src_reg = as_Register($src$$reg);
 8741     Register dst_reg = as_Register($dst$$reg);
 8742     if (dst_reg != src_reg) {
 8743       __ decode_klass_not_null(dst_reg, src_reg);
 8744     } else {
 8745       __ decode_klass_not_null(dst_reg);
 8746     }
 8747   %}
 8748 
 8749    ins_pipe(ialu_reg);
 8750 %}
 8751 
 8752 instruct checkCastPP(iRegPNoSp dst)
 8753 %{
 8754   match(Set dst (CheckCastPP dst));
 8755 
 8756   size(0);
 8757   format %{ "# checkcastPP of $dst" %}
 8758   ins_encode(/* empty encoding */);
 8759   ins_pipe(pipe_class_empty);
 8760 %}
 8761 
 8762 instruct castPP(iRegPNoSp dst)
 8763 %{
 8764   match(Set dst (CastPP dst));
 8765 
 8766   size(0);
 8767   format %{ "# castPP of $dst" %}
 8768   ins_encode(/* empty encoding */);
 8769   ins_pipe(pipe_class_empty);
 8770 %}
 8771 
 8772 instruct castII(iRegI dst)
 8773 %{
 8774   match(Set dst (CastII dst));
 8775 
 8776   size(0);
 8777   format %{ "# castII of $dst" %}
 8778   ins_encode(/* empty encoding */);
 8779   ins_cost(0);
 8780   ins_pipe(pipe_class_empty);
 8781 %}
 8782 
 8783 instruct castLL(iRegL dst)
 8784 %{
 8785   match(Set dst (CastLL dst));
 8786 
 8787   size(0);
 8788   format %{ "# castLL of $dst" %}
 8789   ins_encode(/* empty encoding */);
 8790   ins_cost(0);
 8791   ins_pipe(pipe_class_empty);
 8792 %}
 8793 
 8794 instruct castFF(vRegF dst)
 8795 %{
 8796   match(Set dst (CastFF dst));
 8797 
 8798   size(0);
 8799   format %{ "# castFF of $dst" %}
 8800   ins_encode(/* empty encoding */);
 8801   ins_cost(0);
 8802   ins_pipe(pipe_class_empty);
 8803 %}
 8804 
 8805 instruct castDD(vRegD dst)
 8806 %{
 8807   match(Set dst (CastDD dst));
 8808 
 8809   size(0);
 8810   format %{ "# castDD of $dst" %}
 8811   ins_encode(/* empty encoding */);
 8812   ins_cost(0);
 8813   ins_pipe(pipe_class_empty);
 8814 %}
 8815 
 8816 instruct castVV(vReg dst)
 8817 %{
 8818   match(Set dst (CastVV dst));
 8819 
 8820   size(0);
 8821   format %{ "# castVV of $dst" %}
 8822   ins_encode(/* empty encoding */);
 8823   ins_cost(0);
 8824   ins_pipe(pipe_class_empty);
 8825 %}
 8826 
 8827 instruct castVVMask(pRegGov dst)
 8828 %{
 8829   match(Set dst (CastVV dst));
 8830 
 8831   size(0);
 8832   format %{ "# castVV of $dst" %}
 8833   ins_encode(/* empty encoding */);
 8834   ins_cost(0);
 8835   ins_pipe(pipe_class_empty);
 8836 %}
 8837 
 8838 // ============================================================================
 8839 // Atomic operation instructions
 8840 //
 8841 
 8842 // standard CompareAndSwapX when we are using barriers
 8843 // these have higher priority than the rules selected by a predicate
 8844 
 8845 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8846 // can't match them
 8847 
 8848 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8849 
 8850   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8851   ins_cost(2 * VOLATILE_REF_COST);
 8852 
 8853   effect(KILL cr);
 8854 
 8855   format %{
 8856     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8857     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8858   %}
 8859 
 8860   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8861             aarch64_enc_cset_eq(res));
 8862 
 8863   ins_pipe(pipe_slow);
 8864 %}
 8865 
 8866 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8867 
 8868   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8869   ins_cost(2 * VOLATILE_REF_COST);
 8870 
 8871   effect(KILL cr);
 8872 
 8873   format %{
 8874     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8875     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8876   %}
 8877 
 8878   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8879             aarch64_enc_cset_eq(res));
 8880 
 8881   ins_pipe(pipe_slow);
 8882 %}
 8883 
 8884 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8885 
 8886   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8887   ins_cost(2 * VOLATILE_REF_COST);
 8888 
 8889   effect(KILL cr);
 8890 
 8891  format %{
 8892     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8893     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8894  %}
 8895 
 8896  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8897             aarch64_enc_cset_eq(res));
 8898 
 8899   ins_pipe(pipe_slow);
 8900 %}
 8901 
 8902 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8903 
 8904   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8905   ins_cost(2 * VOLATILE_REF_COST);
 8906 
 8907   effect(KILL cr);
 8908 
 8909  format %{
 8910     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8911     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8912  %}
 8913 
 8914  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8915             aarch64_enc_cset_eq(res));
 8916 
 8917   ins_pipe(pipe_slow);
 8918 %}
 8919 
 8920 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8921 
 8922   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8923   predicate(n->as_LoadStore()->barrier_data() == 0);
 8924   ins_cost(2 * VOLATILE_REF_COST);
 8925 
 8926   effect(KILL cr);
 8927 
 8928  format %{
 8929     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8930     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8931  %}
 8932 
 8933  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8934             aarch64_enc_cset_eq(res));
 8935 
 8936   ins_pipe(pipe_slow);
 8937 %}
 8938 
 8939 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8940 
 8941   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8942   ins_cost(2 * VOLATILE_REF_COST);
 8943 
 8944   effect(KILL cr);
 8945 
 8946  format %{
 8947     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8948     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8949  %}
 8950 
 8951  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8952             aarch64_enc_cset_eq(res));
 8953 
 8954   ins_pipe(pipe_slow);
 8955 %}
 8956 
 8957 // alternative CompareAndSwapX when we are eliding barriers
 8958 
 8959 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8960 
 8961   predicate(needs_acquiring_load_exclusive(n));
 8962   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8963   ins_cost(VOLATILE_REF_COST);
 8964 
 8965   effect(KILL cr);
 8966 
 8967   format %{
 8968     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8969     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8970   %}
 8971 
 8972   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8973             aarch64_enc_cset_eq(res));
 8974 
 8975   ins_pipe(pipe_slow);
 8976 %}
 8977 
 8978 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8979 
 8980   predicate(needs_acquiring_load_exclusive(n));
 8981   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8982   ins_cost(VOLATILE_REF_COST);
 8983 
 8984   effect(KILL cr);
 8985 
 8986   format %{
 8987     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8988     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8989   %}
 8990 
 8991   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8992             aarch64_enc_cset_eq(res));
 8993 
 8994   ins_pipe(pipe_slow);
 8995 %}
 8996 
 8997 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8998 
 8999   predicate(needs_acquiring_load_exclusive(n));
 9000   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 9001   ins_cost(VOLATILE_REF_COST);
 9002 
 9003   effect(KILL cr);
 9004 
 9005  format %{
 9006     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 9007     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9008  %}
 9009 
 9010  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9011             aarch64_enc_cset_eq(res));
 9012 
 9013   ins_pipe(pipe_slow);
 9014 %}
 9015 
 9016 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 9017 
 9018   predicate(needs_acquiring_load_exclusive(n));
 9019   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 9020   ins_cost(VOLATILE_REF_COST);
 9021 
 9022   effect(KILL cr);
 9023 
 9024  format %{
 9025     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 9026     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9027  %}
 9028 
 9029  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9030             aarch64_enc_cset_eq(res));
 9031 
 9032   ins_pipe(pipe_slow);
 9033 %}
 9034 
 9035 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9036 
 9037   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9038   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 9039   ins_cost(VOLATILE_REF_COST);
 9040 
 9041   effect(KILL cr);
 9042 
 9043  format %{
 9044     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 9045     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9046  %}
 9047 
 9048  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 9049             aarch64_enc_cset_eq(res));
 9050 
 9051   ins_pipe(pipe_slow);
 9052 %}
 9053 
 9054 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 9055 
 9056   predicate(needs_acquiring_load_exclusive(n));
 9057   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 9058   ins_cost(VOLATILE_REF_COST);
 9059 
 9060   effect(KILL cr);
 9061 
 9062  format %{
 9063     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 9064     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9065  %}
 9066 
 9067  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 9068             aarch64_enc_cset_eq(res));
 9069 
 9070   ins_pipe(pipe_slow);
 9071 %}
 9072 
 9073 
 9074 // ---------------------------------------------------------------------
 9075 
 9076 // BEGIN This section of the file is automatically generated. Do not edit --------------
 9077 
 9078 // Sundry CAS operations.  Note that release is always true,
 9079 // regardless of the memory ordering of the CAS.  This is because we
 9080 // need the volatile case to be sequentially consistent but there is
 9081 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 9082 // can't check the type of memory ordering here, so we always emit a
 9083 // STLXR.
 9084 
 9085 // This section is generated from cas.m4
 9086 
 9087 
 9088 // This pattern is generated automatically from cas.m4.
 9089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9090 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9091   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9092   ins_cost(2 * VOLATILE_REF_COST);
 9093   effect(TEMP_DEF res, KILL cr);
 9094   format %{
 9095     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9096   %}
 9097   ins_encode %{
 9098     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9099                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9100                /*weak*/ false, $res$$Register);
 9101     __ sxtbw($res$$Register, $res$$Register);
 9102   %}
 9103   ins_pipe(pipe_slow);
 9104 %}
 9105 
 9106 // This pattern is generated automatically from cas.m4.
 9107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9108 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9109   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9110   ins_cost(2 * VOLATILE_REF_COST);
 9111   effect(TEMP_DEF res, KILL cr);
 9112   format %{
 9113     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9114   %}
 9115   ins_encode %{
 9116     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9117                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9118                /*weak*/ false, $res$$Register);
 9119     __ sxthw($res$$Register, $res$$Register);
 9120   %}
 9121   ins_pipe(pipe_slow);
 9122 %}
 9123 
 9124 // This pattern is generated automatically from cas.m4.
 9125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9126 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9127   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9128   ins_cost(2 * VOLATILE_REF_COST);
 9129   effect(TEMP_DEF res, KILL cr);
 9130   format %{
 9131     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9132   %}
 9133   ins_encode %{
 9134     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9135                Assembler::word, /*acquire*/ false, /*release*/ true,
 9136                /*weak*/ false, $res$$Register);
 9137   %}
 9138   ins_pipe(pipe_slow);
 9139 %}
 9140 
 9141 // This pattern is generated automatically from cas.m4.
 9142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9143 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9144   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9145   ins_cost(2 * VOLATILE_REF_COST);
 9146   effect(TEMP_DEF res, KILL cr);
 9147   format %{
 9148     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9149   %}
 9150   ins_encode %{
 9151     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9152                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9153                /*weak*/ false, $res$$Register);
 9154   %}
 9155   ins_pipe(pipe_slow);
 9156 %}
 9157 
 9158 // This pattern is generated automatically from cas.m4.
 9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9160 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9161   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9162   ins_cost(2 * VOLATILE_REF_COST);
 9163   effect(TEMP_DEF res, KILL cr);
 9164   format %{
 9165     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9166   %}
 9167   ins_encode %{
 9168     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9169                Assembler::word, /*acquire*/ false, /*release*/ true,
 9170                /*weak*/ false, $res$$Register);
 9171   %}
 9172   ins_pipe(pipe_slow);
 9173 %}
 9174 
 9175 // This pattern is generated automatically from cas.m4.
 9176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9177 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9178   predicate(n->as_LoadStore()->barrier_data() == 0);
 9179   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9180   ins_cost(2 * VOLATILE_REF_COST);
 9181   effect(TEMP_DEF res, KILL cr);
 9182   format %{
 9183     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9184   %}
 9185   ins_encode %{
 9186     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9187                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9188                /*weak*/ false, $res$$Register);
 9189   %}
 9190   ins_pipe(pipe_slow);
 9191 %}
 9192 
 9193 // This pattern is generated automatically from cas.m4.
 9194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9195 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9196   predicate(needs_acquiring_load_exclusive(n));
 9197   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9198   ins_cost(VOLATILE_REF_COST);
 9199   effect(TEMP_DEF res, KILL cr);
 9200   format %{
 9201     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9202   %}
 9203   ins_encode %{
 9204     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9205                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9206                /*weak*/ false, $res$$Register);
 9207     __ sxtbw($res$$Register, $res$$Register);
 9208   %}
 9209   ins_pipe(pipe_slow);
 9210 %}
 9211 
 9212 // This pattern is generated automatically from cas.m4.
 9213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9214 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9215   predicate(needs_acquiring_load_exclusive(n));
 9216   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9217   ins_cost(VOLATILE_REF_COST);
 9218   effect(TEMP_DEF res, KILL cr);
 9219   format %{
 9220     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9221   %}
 9222   ins_encode %{
 9223     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9224                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9225                /*weak*/ false, $res$$Register);
 9226     __ sxthw($res$$Register, $res$$Register);
 9227   %}
 9228   ins_pipe(pipe_slow);
 9229 %}
 9230 
 9231 // This pattern is generated automatically from cas.m4.
 9232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9233 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9234   predicate(needs_acquiring_load_exclusive(n));
 9235   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9236   ins_cost(VOLATILE_REF_COST);
 9237   effect(TEMP_DEF res, KILL cr);
 9238   format %{
 9239     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9240   %}
 9241   ins_encode %{
 9242     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9243                Assembler::word, /*acquire*/ true, /*release*/ true,
 9244                /*weak*/ false, $res$$Register);
 9245   %}
 9246   ins_pipe(pipe_slow);
 9247 %}
 9248 
 9249 // This pattern is generated automatically from cas.m4.
 9250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9251 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9252   predicate(needs_acquiring_load_exclusive(n));
 9253   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9254   ins_cost(VOLATILE_REF_COST);
 9255   effect(TEMP_DEF res, KILL cr);
 9256   format %{
 9257     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9258   %}
 9259   ins_encode %{
 9260     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9261                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9262                /*weak*/ false, $res$$Register);
 9263   %}
 9264   ins_pipe(pipe_slow);
 9265 %}
 9266 
 9267 // This pattern is generated automatically from cas.m4.
 9268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9269 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9270   predicate(needs_acquiring_load_exclusive(n));
 9271   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9272   ins_cost(VOLATILE_REF_COST);
 9273   effect(TEMP_DEF res, KILL cr);
 9274   format %{
 9275     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9276   %}
 9277   ins_encode %{
 9278     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9279                Assembler::word, /*acquire*/ true, /*release*/ true,
 9280                /*weak*/ false, $res$$Register);
 9281   %}
 9282   ins_pipe(pipe_slow);
 9283 %}
 9284 
 9285 // This pattern is generated automatically from cas.m4.
 9286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9287 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9288   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9289   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9290   ins_cost(VOLATILE_REF_COST);
 9291   effect(TEMP_DEF res, KILL cr);
 9292   format %{
 9293     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9294   %}
 9295   ins_encode %{
 9296     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9297                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9298                /*weak*/ false, $res$$Register);
 9299   %}
 9300   ins_pipe(pipe_slow);
 9301 %}
 9302 
 9303 // This pattern is generated automatically from cas.m4.
 9304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9305 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9306   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9307   ins_cost(2 * VOLATILE_REF_COST);
 9308   effect(KILL cr);
 9309   format %{
 9310     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9311     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9312   %}
 9313   ins_encode %{
 9314     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9315                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9316                /*weak*/ true, noreg);
 9317     __ csetw($res$$Register, Assembler::EQ);
 9318   %}
 9319   ins_pipe(pipe_slow);
 9320 %}
 9321 
 9322 // This pattern is generated automatically from cas.m4.
 9323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9324 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9325   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9326   ins_cost(2 * VOLATILE_REF_COST);
 9327   effect(KILL cr);
 9328   format %{
 9329     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9330     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9331   %}
 9332   ins_encode %{
 9333     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9334                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9335                /*weak*/ true, noreg);
 9336     __ csetw($res$$Register, Assembler::EQ);
 9337   %}
 9338   ins_pipe(pipe_slow);
 9339 %}
 9340 
 9341 // This pattern is generated automatically from cas.m4.
 9342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9343 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9344   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9345   ins_cost(2 * VOLATILE_REF_COST);
 9346   effect(KILL cr);
 9347   format %{
 9348     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9349     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9350   %}
 9351   ins_encode %{
 9352     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9353                Assembler::word, /*acquire*/ false, /*release*/ true,
 9354                /*weak*/ true, noreg);
 9355     __ csetw($res$$Register, Assembler::EQ);
 9356   %}
 9357   ins_pipe(pipe_slow);
 9358 %}
 9359 
 9360 // This pattern is generated automatically from cas.m4.
 9361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9362 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9363   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9364   ins_cost(2 * VOLATILE_REF_COST);
 9365   effect(KILL cr);
 9366   format %{
 9367     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9368     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9369   %}
 9370   ins_encode %{
 9371     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9372                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9373                /*weak*/ true, noreg);
 9374     __ csetw($res$$Register, Assembler::EQ);
 9375   %}
 9376   ins_pipe(pipe_slow);
 9377 %}
 9378 
 9379 // This pattern is generated automatically from cas.m4.
 9380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9381 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9382   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9383   ins_cost(2 * VOLATILE_REF_COST);
 9384   effect(KILL cr);
 9385   format %{
 9386     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9387     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9388   %}
 9389   ins_encode %{
 9390     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9391                Assembler::word, /*acquire*/ false, /*release*/ true,
 9392                /*weak*/ true, noreg);
 9393     __ csetw($res$$Register, Assembler::EQ);
 9394   %}
 9395   ins_pipe(pipe_slow);
 9396 %}
 9397 
 9398 // This pattern is generated automatically from cas.m4.
 9399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9400 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9401   predicate(n->as_LoadStore()->barrier_data() == 0);
 9402   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9403   ins_cost(2 * VOLATILE_REF_COST);
 9404   effect(KILL cr);
 9405   format %{
 9406     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9407     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9408   %}
 9409   ins_encode %{
 9410     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9411                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9412                /*weak*/ true, noreg);
 9413     __ csetw($res$$Register, Assembler::EQ);
 9414   %}
 9415   ins_pipe(pipe_slow);
 9416 %}
 9417 
 9418 // This pattern is generated automatically from cas.m4.
 9419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9420 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9421   predicate(needs_acquiring_load_exclusive(n));
 9422   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9423   ins_cost(VOLATILE_REF_COST);
 9424   effect(KILL cr);
 9425   format %{
 9426     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9427     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9428   %}
 9429   ins_encode %{
 9430     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9431                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9432                /*weak*/ true, noreg);
 9433     __ csetw($res$$Register, Assembler::EQ);
 9434   %}
 9435   ins_pipe(pipe_slow);
 9436 %}
 9437 
 9438 // This pattern is generated automatically from cas.m4.
 9439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9440 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9441   predicate(needs_acquiring_load_exclusive(n));
 9442   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9443   ins_cost(VOLATILE_REF_COST);
 9444   effect(KILL cr);
 9445   format %{
 9446     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9447     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9448   %}
 9449   ins_encode %{
 9450     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9451                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9452                /*weak*/ true, noreg);
 9453     __ csetw($res$$Register, Assembler::EQ);
 9454   %}
 9455   ins_pipe(pipe_slow);
 9456 %}
 9457 
 9458 // This pattern is generated automatically from cas.m4.
 9459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9460 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9461   predicate(needs_acquiring_load_exclusive(n));
 9462   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9463   ins_cost(VOLATILE_REF_COST);
 9464   effect(KILL cr);
 9465   format %{
 9466     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9467     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9468   %}
 9469   ins_encode %{
 9470     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9471                Assembler::word, /*acquire*/ true, /*release*/ true,
 9472                /*weak*/ true, noreg);
 9473     __ csetw($res$$Register, Assembler::EQ);
 9474   %}
 9475   ins_pipe(pipe_slow);
 9476 %}
 9477 
 9478 // This pattern is generated automatically from cas.m4.
 9479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9480 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9481   predicate(needs_acquiring_load_exclusive(n));
 9482   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9483   ins_cost(VOLATILE_REF_COST);
 9484   effect(KILL cr);
 9485   format %{
 9486     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9487     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9488   %}
 9489   ins_encode %{
 9490     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9491                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9492                /*weak*/ true, noreg);
 9493     __ csetw($res$$Register, Assembler::EQ);
 9494   %}
 9495   ins_pipe(pipe_slow);
 9496 %}
 9497 
 9498 // This pattern is generated automatically from cas.m4.
 9499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9500 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9501   predicate(needs_acquiring_load_exclusive(n));
 9502   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9503   ins_cost(VOLATILE_REF_COST);
 9504   effect(KILL cr);
 9505   format %{
 9506     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9507     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9508   %}
 9509   ins_encode %{
 9510     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9511                Assembler::word, /*acquire*/ true, /*release*/ true,
 9512                /*weak*/ true, noreg);
 9513     __ csetw($res$$Register, Assembler::EQ);
 9514   %}
 9515   ins_pipe(pipe_slow);
 9516 %}
 9517 
 9518 // This pattern is generated automatically from cas.m4.
 9519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9520 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9521   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9522   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9523   ins_cost(VOLATILE_REF_COST);
 9524   effect(KILL cr);
 9525   format %{
 9526     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9527     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9528   %}
 9529   ins_encode %{
 9530     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9531                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9532                /*weak*/ true, noreg);
 9533     __ csetw($res$$Register, Assembler::EQ);
 9534   %}
 9535   ins_pipe(pipe_slow);
 9536 %}
 9537 
 9538 // END This section of the file is automatically generated. Do not edit --------------
 9539 // ---------------------------------------------------------------------
 9540 
 9541 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9542   match(Set prev (GetAndSetI mem newv));
 9543   ins_cost(2 * VOLATILE_REF_COST);
 9544   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9545   ins_encode %{
 9546     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9547   %}
 9548   ins_pipe(pipe_serial);
 9549 %}
 9550 
 9551 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9552   match(Set prev (GetAndSetL mem newv));
 9553   ins_cost(2 * VOLATILE_REF_COST);
 9554   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9555   ins_encode %{
 9556     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9557   %}
 9558   ins_pipe(pipe_serial);
 9559 %}
 9560 
 9561 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9562   match(Set prev (GetAndSetN mem newv));
 9563   ins_cost(2 * VOLATILE_REF_COST);
 9564   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9565   ins_encode %{
 9566     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9567   %}
 9568   ins_pipe(pipe_serial);
 9569 %}
 9570 
 9571 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9572   predicate(n->as_LoadStore()->barrier_data() == 0);
 9573   match(Set prev (GetAndSetP mem newv));
 9574   ins_cost(2 * VOLATILE_REF_COST);
 9575   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9576   ins_encode %{
 9577     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9578   %}
 9579   ins_pipe(pipe_serial);
 9580 %}
 9581 
 9582 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9583   predicate(needs_acquiring_load_exclusive(n));
 9584   match(Set prev (GetAndSetI mem newv));
 9585   ins_cost(VOLATILE_REF_COST);
 9586   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9587   ins_encode %{
 9588     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9589   %}
 9590   ins_pipe(pipe_serial);
 9591 %}
 9592 
 9593 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9594   predicate(needs_acquiring_load_exclusive(n));
 9595   match(Set prev (GetAndSetL mem newv));
 9596   ins_cost(VOLATILE_REF_COST);
 9597   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9598   ins_encode %{
 9599     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9600   %}
 9601   ins_pipe(pipe_serial);
 9602 %}
 9603 
 9604 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9605   predicate(needs_acquiring_load_exclusive(n));
 9606   match(Set prev (GetAndSetN mem newv));
 9607   ins_cost(VOLATILE_REF_COST);
 9608   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9609   ins_encode %{
 9610     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9611   %}
 9612   ins_pipe(pipe_serial);
 9613 %}
 9614 
 9615 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9616   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9617   match(Set prev (GetAndSetP mem newv));
 9618   ins_cost(VOLATILE_REF_COST);
 9619   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9620   ins_encode %{
 9621     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9622   %}
 9623   ins_pipe(pipe_serial);
 9624 %}
 9625 
 9626 
 9627 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9628   match(Set newval (GetAndAddL mem incr));
 9629   ins_cost(2 * VOLATILE_REF_COST + 1);
 9630   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9631   ins_encode %{
 9632     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9633   %}
 9634   ins_pipe(pipe_serial);
 9635 %}
 9636 
 9637 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9638   predicate(n->as_LoadStore()->result_not_used());
 9639   match(Set dummy (GetAndAddL mem incr));
 9640   ins_cost(2 * VOLATILE_REF_COST);
 9641   format %{ "get_and_addL [$mem], $incr" %}
 9642   ins_encode %{
 9643     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9644   %}
 9645   ins_pipe(pipe_serial);
 9646 %}
 9647 
 9648 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9649   match(Set newval (GetAndAddL mem incr));
 9650   ins_cost(2 * VOLATILE_REF_COST + 1);
 9651   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9652   ins_encode %{
 9653     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9654   %}
 9655   ins_pipe(pipe_serial);
 9656 %}
 9657 
 9658 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9659   predicate(n->as_LoadStore()->result_not_used());
 9660   match(Set dummy (GetAndAddL mem incr));
 9661   ins_cost(2 * VOLATILE_REF_COST);
 9662   format %{ "get_and_addL [$mem], $incr" %}
 9663   ins_encode %{
 9664     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9665   %}
 9666   ins_pipe(pipe_serial);
 9667 %}
 9668 
 9669 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9670   match(Set newval (GetAndAddI mem incr));
 9671   ins_cost(2 * VOLATILE_REF_COST + 1);
 9672   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9673   ins_encode %{
 9674     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9675   %}
 9676   ins_pipe(pipe_serial);
 9677 %}
 9678 
 9679 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9680   predicate(n->as_LoadStore()->result_not_used());
 9681   match(Set dummy (GetAndAddI mem incr));
 9682   ins_cost(2 * VOLATILE_REF_COST);
 9683   format %{ "get_and_addI [$mem], $incr" %}
 9684   ins_encode %{
 9685     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9686   %}
 9687   ins_pipe(pipe_serial);
 9688 %}
 9689 
 9690 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9691   match(Set newval (GetAndAddI mem incr));
 9692   ins_cost(2 * VOLATILE_REF_COST + 1);
 9693   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9694   ins_encode %{
 9695     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9696   %}
 9697   ins_pipe(pipe_serial);
 9698 %}
 9699 
 9700 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9701   predicate(n->as_LoadStore()->result_not_used());
 9702   match(Set dummy (GetAndAddI mem incr));
 9703   ins_cost(2 * VOLATILE_REF_COST);
 9704   format %{ "get_and_addI [$mem], $incr" %}
 9705   ins_encode %{
 9706     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9707   %}
 9708   ins_pipe(pipe_serial);
 9709 %}
 9710 
 9711 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9712   predicate(needs_acquiring_load_exclusive(n));
 9713   match(Set newval (GetAndAddL mem incr));
 9714   ins_cost(VOLATILE_REF_COST + 1);
 9715   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9716   ins_encode %{
 9717     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9718   %}
 9719   ins_pipe(pipe_serial);
 9720 %}
 9721 
 9722 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9723   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9724   match(Set dummy (GetAndAddL mem incr));
 9725   ins_cost(VOLATILE_REF_COST);
 9726   format %{ "get_and_addL_acq [$mem], $incr" %}
 9727   ins_encode %{
 9728     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9729   %}
 9730   ins_pipe(pipe_serial);
 9731 %}
 9732 
 9733 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9734   predicate(needs_acquiring_load_exclusive(n));
 9735   match(Set newval (GetAndAddL mem incr));
 9736   ins_cost(VOLATILE_REF_COST + 1);
 9737   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9738   ins_encode %{
 9739     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9740   %}
 9741   ins_pipe(pipe_serial);
 9742 %}
 9743 
 9744 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9745   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9746   match(Set dummy (GetAndAddL mem incr));
 9747   ins_cost(VOLATILE_REF_COST);
 9748   format %{ "get_and_addL_acq [$mem], $incr" %}
 9749   ins_encode %{
 9750     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9751   %}
 9752   ins_pipe(pipe_serial);
 9753 %}
 9754 
 9755 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9756   predicate(needs_acquiring_load_exclusive(n));
 9757   match(Set newval (GetAndAddI mem incr));
 9758   ins_cost(VOLATILE_REF_COST + 1);
 9759   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9760   ins_encode %{
 9761     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9762   %}
 9763   ins_pipe(pipe_serial);
 9764 %}
 9765 
 9766 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9767   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9768   match(Set dummy (GetAndAddI mem incr));
 9769   ins_cost(VOLATILE_REF_COST);
 9770   format %{ "get_and_addI_acq [$mem], $incr" %}
 9771   ins_encode %{
 9772     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9773   %}
 9774   ins_pipe(pipe_serial);
 9775 %}
 9776 
 9777 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9778   predicate(needs_acquiring_load_exclusive(n));
 9779   match(Set newval (GetAndAddI mem incr));
 9780   ins_cost(VOLATILE_REF_COST + 1);
 9781   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9782   ins_encode %{
 9783     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9784   %}
 9785   ins_pipe(pipe_serial);
 9786 %}
 9787 
 9788 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9789   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9790   match(Set dummy (GetAndAddI mem incr));
 9791   ins_cost(VOLATILE_REF_COST);
 9792   format %{ "get_and_addI_acq [$mem], $incr" %}
 9793   ins_encode %{
 9794     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9795   %}
 9796   ins_pipe(pipe_serial);
 9797 %}
 9798 
 9799 // Manifest a CmpU result in an integer register.
 9800 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9801 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9802 %{
 9803   match(Set dst (CmpU3 src1 src2));
 9804   effect(KILL flags);
 9805 
 9806   ins_cost(INSN_COST * 3);
 9807   format %{
 9808       "cmpw $src1, $src2\n\t"
 9809       "csetw $dst, ne\n\t"
 9810       "cnegw $dst, lo\t# CmpU3(reg)"
 9811   %}
 9812   ins_encode %{
 9813     __ cmpw($src1$$Register, $src2$$Register);
 9814     __ csetw($dst$$Register, Assembler::NE);
 9815     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9816   %}
 9817 
 9818   ins_pipe(pipe_class_default);
 9819 %}
 9820 
 9821 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9822 %{
 9823   match(Set dst (CmpU3 src1 src2));
 9824   effect(KILL flags);
 9825 
 9826   ins_cost(INSN_COST * 3);
 9827   format %{
 9828       "subsw zr, $src1, $src2\n\t"
 9829       "csetw $dst, ne\n\t"
 9830       "cnegw $dst, lo\t# CmpU3(imm)"
 9831   %}
 9832   ins_encode %{
 9833     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9834     __ csetw($dst$$Register, Assembler::NE);
 9835     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9836   %}
 9837 
 9838   ins_pipe(pipe_class_default);
 9839 %}
 9840 
 9841 // Manifest a CmpUL result in an integer register.
 9842 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9843 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9844 %{
 9845   match(Set dst (CmpUL3 src1 src2));
 9846   effect(KILL flags);
 9847 
 9848   ins_cost(INSN_COST * 3);
 9849   format %{
 9850       "cmp $src1, $src2\n\t"
 9851       "csetw $dst, ne\n\t"
 9852       "cnegw $dst, lo\t# CmpUL3(reg)"
 9853   %}
 9854   ins_encode %{
 9855     __ cmp($src1$$Register, $src2$$Register);
 9856     __ csetw($dst$$Register, Assembler::NE);
 9857     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9858   %}
 9859 
 9860   ins_pipe(pipe_class_default);
 9861 %}
 9862 
 9863 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9864 %{
 9865   match(Set dst (CmpUL3 src1 src2));
 9866   effect(KILL flags);
 9867 
 9868   ins_cost(INSN_COST * 3);
 9869   format %{
 9870       "subs zr, $src1, $src2\n\t"
 9871       "csetw $dst, ne\n\t"
 9872       "cnegw $dst, lo\t# CmpUL3(imm)"
 9873   %}
 9874   ins_encode %{
 9875     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9876     __ csetw($dst$$Register, Assembler::NE);
 9877     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9878   %}
 9879 
 9880   ins_pipe(pipe_class_default);
 9881 %}
 9882 
 9883 // Manifest a CmpL result in an integer register.
 9884 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9885 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9886 %{
 9887   match(Set dst (CmpL3 src1 src2));
 9888   effect(KILL flags);
 9889 
 9890   ins_cost(INSN_COST * 3);
 9891   format %{
 9892       "cmp $src1, $src2\n\t"
 9893       "csetw $dst, ne\n\t"
 9894       "cnegw $dst, lt\t# CmpL3(reg)"
 9895   %}
 9896   ins_encode %{
 9897     __ cmp($src1$$Register, $src2$$Register);
 9898     __ csetw($dst$$Register, Assembler::NE);
 9899     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9900   %}
 9901 
 9902   ins_pipe(pipe_class_default);
 9903 %}
 9904 
 9905 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9906 %{
 9907   match(Set dst (CmpL3 src1 src2));
 9908   effect(KILL flags);
 9909 
 9910   ins_cost(INSN_COST * 3);
 9911   format %{
 9912       "subs zr, $src1, $src2\n\t"
 9913       "csetw $dst, ne\n\t"
 9914       "cnegw $dst, lt\t# CmpL3(imm)"
 9915   %}
 9916   ins_encode %{
 9917     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9918     __ csetw($dst$$Register, Assembler::NE);
 9919     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9920   %}
 9921 
 9922   ins_pipe(pipe_class_default);
 9923 %}
 9924 
 9925 // ============================================================================
 9926 // Conditional Move Instructions
 9927 
 9928 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9929 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9930 // define an op class which merged both inputs and use it to type the
 9931 // argument to a single rule. unfortunatelyt his fails because the
 9932 // opclass does not live up to the COND_INTER interface of its
 9933 // component operands. When the generic code tries to negate the
 9934 // operand it ends up running the generci Machoper::negate method
 9935 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9936 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9937 
 9938 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9939   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9940 
 9941   ins_cost(INSN_COST * 2);
 9942   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9943 
 9944   ins_encode %{
 9945     __ cselw(as_Register($dst$$reg),
 9946              as_Register($src2$$reg),
 9947              as_Register($src1$$reg),
 9948              (Assembler::Condition)$cmp$$cmpcode);
 9949   %}
 9950 
 9951   ins_pipe(icond_reg_reg);
 9952 %}
 9953 
 9954 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9955   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9956 
 9957   ins_cost(INSN_COST * 2);
 9958   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9959 
 9960   ins_encode %{
 9961     __ cselw(as_Register($dst$$reg),
 9962              as_Register($src2$$reg),
 9963              as_Register($src1$$reg),
 9964              (Assembler::Condition)$cmp$$cmpcode);
 9965   %}
 9966 
 9967   ins_pipe(icond_reg_reg);
 9968 %}
 9969 
 9970 // special cases where one arg is zero
 9971 
 9972 // n.b. this is selected in preference to the rule above because it
 9973 // avoids loading constant 0 into a source register
 9974 
 9975 // TODO
 9976 // we ought only to be able to cull one of these variants as the ideal
 9977 // transforms ought always to order the zero consistently (to left/right?)
 9978 
 9979 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9980   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9981 
 9982   ins_cost(INSN_COST * 2);
 9983   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9984 
 9985   ins_encode %{
 9986     __ cselw(as_Register($dst$$reg),
 9987              as_Register($src$$reg),
 9988              zr,
 9989              (Assembler::Condition)$cmp$$cmpcode);
 9990   %}
 9991 
 9992   ins_pipe(icond_reg);
 9993 %}
 9994 
 9995 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9996   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9997 
 9998   ins_cost(INSN_COST * 2);
 9999   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
10000 
10001   ins_encode %{
10002     __ cselw(as_Register($dst$$reg),
10003              as_Register($src$$reg),
10004              zr,
10005              (Assembler::Condition)$cmp$$cmpcode);
10006   %}
10007 
10008   ins_pipe(icond_reg);
10009 %}
10010 
10011 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10012   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10013 
10014   ins_cost(INSN_COST * 2);
10015   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
10016 
10017   ins_encode %{
10018     __ cselw(as_Register($dst$$reg),
10019              zr,
10020              as_Register($src$$reg),
10021              (Assembler::Condition)$cmp$$cmpcode);
10022   %}
10023 
10024   ins_pipe(icond_reg);
10025 %}
10026 
10027 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10028   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10029 
10030   ins_cost(INSN_COST * 2);
10031   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
10032 
10033   ins_encode %{
10034     __ cselw(as_Register($dst$$reg),
10035              zr,
10036              as_Register($src$$reg),
10037              (Assembler::Condition)$cmp$$cmpcode);
10038   %}
10039 
10040   ins_pipe(icond_reg);
10041 %}
10042 
10043 // special case for creating a boolean 0 or 1
10044 
10045 // n.b. this is selected in preference to the rule above because it
10046 // avoids loading constants 0 and 1 into a source register
10047 
10048 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10049   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10050 
10051   ins_cost(INSN_COST * 2);
10052   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
10053 
10054   ins_encode %{
10055     // equivalently
10056     // cset(as_Register($dst$$reg),
10057     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10058     __ csincw(as_Register($dst$$reg),
10059              zr,
10060              zr,
10061              (Assembler::Condition)$cmp$$cmpcode);
10062   %}
10063 
10064   ins_pipe(icond_none);
10065 %}
10066 
10067 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10068   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10069 
10070   ins_cost(INSN_COST * 2);
10071   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
10072 
10073   ins_encode %{
10074     // equivalently
10075     // cset(as_Register($dst$$reg),
10076     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
10077     __ csincw(as_Register($dst$$reg),
10078              zr,
10079              zr,
10080              (Assembler::Condition)$cmp$$cmpcode);
10081   %}
10082 
10083   ins_pipe(icond_none);
10084 %}
10085 
10086 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10087   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10088 
10089   ins_cost(INSN_COST * 2);
10090   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10091 
10092   ins_encode %{
10093     __ csel(as_Register($dst$$reg),
10094             as_Register($src2$$reg),
10095             as_Register($src1$$reg),
10096             (Assembler::Condition)$cmp$$cmpcode);
10097   %}
10098 
10099   ins_pipe(icond_reg_reg);
10100 %}
10101 
10102 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10103   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10104 
10105   ins_cost(INSN_COST * 2);
10106   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10107 
10108   ins_encode %{
10109     __ csel(as_Register($dst$$reg),
10110             as_Register($src2$$reg),
10111             as_Register($src1$$reg),
10112             (Assembler::Condition)$cmp$$cmpcode);
10113   %}
10114 
10115   ins_pipe(icond_reg_reg);
10116 %}
10117 
10118 // special cases where one arg is zero
10119 
10120 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10121   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10122 
10123   ins_cost(INSN_COST * 2);
10124   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10125 
10126   ins_encode %{
10127     __ csel(as_Register($dst$$reg),
10128             zr,
10129             as_Register($src$$reg),
10130             (Assembler::Condition)$cmp$$cmpcode);
10131   %}
10132 
10133   ins_pipe(icond_reg);
10134 %}
10135 
10136 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10137   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10138 
10139   ins_cost(INSN_COST * 2);
10140   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10141 
10142   ins_encode %{
10143     __ csel(as_Register($dst$$reg),
10144             zr,
10145             as_Register($src$$reg),
10146             (Assembler::Condition)$cmp$$cmpcode);
10147   %}
10148 
10149   ins_pipe(icond_reg);
10150 %}
10151 
10152 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10153   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10154 
10155   ins_cost(INSN_COST * 2);
10156   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10157 
10158   ins_encode %{
10159     __ csel(as_Register($dst$$reg),
10160             as_Register($src$$reg),
10161             zr,
10162             (Assembler::Condition)$cmp$$cmpcode);
10163   %}
10164 
10165   ins_pipe(icond_reg);
10166 %}
10167 
10168 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10169   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10170 
10171   ins_cost(INSN_COST * 2);
10172   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10173 
10174   ins_encode %{
10175     __ csel(as_Register($dst$$reg),
10176             as_Register($src$$reg),
10177             zr,
10178             (Assembler::Condition)$cmp$$cmpcode);
10179   %}
10180 
10181   ins_pipe(icond_reg);
10182 %}
10183 
10184 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10185   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10186 
10187   ins_cost(INSN_COST * 2);
10188   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10189 
10190   ins_encode %{
10191     __ csel(as_Register($dst$$reg),
10192             as_Register($src2$$reg),
10193             as_Register($src1$$reg),
10194             (Assembler::Condition)$cmp$$cmpcode);
10195   %}
10196 
10197   ins_pipe(icond_reg_reg);
10198 %}
10199 
10200 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10201   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10202 
10203   ins_cost(INSN_COST * 2);
10204   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10205 
10206   ins_encode %{
10207     __ csel(as_Register($dst$$reg),
10208             as_Register($src2$$reg),
10209             as_Register($src1$$reg),
10210             (Assembler::Condition)$cmp$$cmpcode);
10211   %}
10212 
10213   ins_pipe(icond_reg_reg);
10214 %}
10215 
10216 // special cases where one arg is zero
10217 
10218 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10219   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10220 
10221   ins_cost(INSN_COST * 2);
10222   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10223 
10224   ins_encode %{
10225     __ csel(as_Register($dst$$reg),
10226             zr,
10227             as_Register($src$$reg),
10228             (Assembler::Condition)$cmp$$cmpcode);
10229   %}
10230 
10231   ins_pipe(icond_reg);
10232 %}
10233 
10234 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10235   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10236 
10237   ins_cost(INSN_COST * 2);
10238   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10239 
10240   ins_encode %{
10241     __ csel(as_Register($dst$$reg),
10242             zr,
10243             as_Register($src$$reg),
10244             (Assembler::Condition)$cmp$$cmpcode);
10245   %}
10246 
10247   ins_pipe(icond_reg);
10248 %}
10249 
10250 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10251   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10252 
10253   ins_cost(INSN_COST * 2);
10254   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10255 
10256   ins_encode %{
10257     __ csel(as_Register($dst$$reg),
10258             as_Register($src$$reg),
10259             zr,
10260             (Assembler::Condition)$cmp$$cmpcode);
10261   %}
10262 
10263   ins_pipe(icond_reg);
10264 %}
10265 
10266 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10267   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10268 
10269   ins_cost(INSN_COST * 2);
10270   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10271 
10272   ins_encode %{
10273     __ csel(as_Register($dst$$reg),
10274             as_Register($src$$reg),
10275             zr,
10276             (Assembler::Condition)$cmp$$cmpcode);
10277   %}
10278 
10279   ins_pipe(icond_reg);
10280 %}
10281 
10282 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10283   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10284 
10285   ins_cost(INSN_COST * 2);
10286   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10287 
10288   ins_encode %{
10289     __ cselw(as_Register($dst$$reg),
10290              as_Register($src2$$reg),
10291              as_Register($src1$$reg),
10292              (Assembler::Condition)$cmp$$cmpcode);
10293   %}
10294 
10295   ins_pipe(icond_reg_reg);
10296 %}
10297 
10298 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10299   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10300 
10301   ins_cost(INSN_COST * 2);
10302   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10303 
10304   ins_encode %{
10305     __ cselw(as_Register($dst$$reg),
10306              as_Register($src2$$reg),
10307              as_Register($src1$$reg),
10308              (Assembler::Condition)$cmp$$cmpcode);
10309   %}
10310 
10311   ins_pipe(icond_reg_reg);
10312 %}
10313 
10314 // special cases where one arg is zero
10315 
10316 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10317   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10321 
10322   ins_encode %{
10323     __ cselw(as_Register($dst$$reg),
10324              zr,
10325              as_Register($src$$reg),
10326              (Assembler::Condition)$cmp$$cmpcode);
10327   %}
10328 
10329   ins_pipe(icond_reg);
10330 %}
10331 
10332 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10333   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10334 
10335   ins_cost(INSN_COST * 2);
10336   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10337 
10338   ins_encode %{
10339     __ cselw(as_Register($dst$$reg),
10340              zr,
10341              as_Register($src$$reg),
10342              (Assembler::Condition)$cmp$$cmpcode);
10343   %}
10344 
10345   ins_pipe(icond_reg);
10346 %}
10347 
10348 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10349   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10350 
10351   ins_cost(INSN_COST * 2);
10352   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10353 
10354   ins_encode %{
10355     __ cselw(as_Register($dst$$reg),
10356              as_Register($src$$reg),
10357              zr,
10358              (Assembler::Condition)$cmp$$cmpcode);
10359   %}
10360 
10361   ins_pipe(icond_reg);
10362 %}
10363 
10364 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10365   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10366 
10367   ins_cost(INSN_COST * 2);
10368   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10369 
10370   ins_encode %{
10371     __ cselw(as_Register($dst$$reg),
10372              as_Register($src$$reg),
10373              zr,
10374              (Assembler::Condition)$cmp$$cmpcode);
10375   %}
10376 
10377   ins_pipe(icond_reg);
10378 %}
10379 
10380 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10381 %{
10382   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10383 
10384   ins_cost(INSN_COST * 3);
10385 
10386   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10387   ins_encode %{
10388     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10389     __ fcsels(as_FloatRegister($dst$$reg),
10390               as_FloatRegister($src2$$reg),
10391               as_FloatRegister($src1$$reg),
10392               cond);
10393   %}
10394 
10395   ins_pipe(fp_cond_reg_reg_s);
10396 %}
10397 
10398 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10399 %{
10400   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10401 
10402   ins_cost(INSN_COST * 3);
10403 
10404   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10405   ins_encode %{
10406     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10407     __ fcsels(as_FloatRegister($dst$$reg),
10408               as_FloatRegister($src2$$reg),
10409               as_FloatRegister($src1$$reg),
10410               cond);
10411   %}
10412 
10413   ins_pipe(fp_cond_reg_reg_s);
10414 %}
10415 
10416 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10417 %{
10418   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10419 
10420   ins_cost(INSN_COST * 3);
10421 
10422   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10423   ins_encode %{
10424     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10425     __ fcseld(as_FloatRegister($dst$$reg),
10426               as_FloatRegister($src2$$reg),
10427               as_FloatRegister($src1$$reg),
10428               cond);
10429   %}
10430 
10431   ins_pipe(fp_cond_reg_reg_d);
10432 %}
10433 
10434 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10435 %{
10436   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10437 
10438   ins_cost(INSN_COST * 3);
10439 
10440   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10441   ins_encode %{
10442     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10443     __ fcseld(as_FloatRegister($dst$$reg),
10444               as_FloatRegister($src2$$reg),
10445               as_FloatRegister($src1$$reg),
10446               cond);
10447   %}
10448 
10449   ins_pipe(fp_cond_reg_reg_d);
10450 %}
10451 
10452 // ============================================================================
10453 // Arithmetic Instructions
10454 //
10455 
10456 // Integer Addition
10457 
10458 // TODO
10459 // these currently employ operations which do not set CR and hence are
10460 // not flagged as killing CR but we would like to isolate the cases
10461 // where we want to set flags from those where we don't. need to work
10462 // out how to do that.
10463 
10464 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10465   match(Set dst (AddI src1 src2));
10466 
10467   ins_cost(INSN_COST);
10468   format %{ "addw  $dst, $src1, $src2" %}
10469 
10470   ins_encode %{
10471     __ addw(as_Register($dst$$reg),
10472             as_Register($src1$$reg),
10473             as_Register($src2$$reg));
10474   %}
10475 
10476   ins_pipe(ialu_reg_reg);
10477 %}
10478 
10479 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10480   match(Set dst (AddI src1 src2));
10481 
10482   ins_cost(INSN_COST);
10483   format %{ "addw $dst, $src1, $src2" %}
10484 
10485   // use opcode to indicate that this is an add not a sub
10486   opcode(0x0);
10487 
10488   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10489 
10490   ins_pipe(ialu_reg_imm);
10491 %}
10492 
10493 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10494   match(Set dst (AddI (ConvL2I src1) src2));
10495 
10496   ins_cost(INSN_COST);
10497   format %{ "addw $dst, $src1, $src2" %}
10498 
10499   // use opcode to indicate that this is an add not a sub
10500   opcode(0x0);
10501 
10502   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10503 
10504   ins_pipe(ialu_reg_imm);
10505 %}
10506 
10507 // Pointer Addition
10508 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10509   match(Set dst (AddP src1 src2));
10510 
10511   ins_cost(INSN_COST);
10512   format %{ "add $dst, $src1, $src2\t# ptr" %}
10513 
10514   ins_encode %{
10515     __ add(as_Register($dst$$reg),
10516            as_Register($src1$$reg),
10517            as_Register($src2$$reg));
10518   %}
10519 
10520   ins_pipe(ialu_reg_reg);
10521 %}
10522 
10523 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10524   match(Set dst (AddP src1 (ConvI2L src2)));
10525 
10526   ins_cost(1.9 * INSN_COST);
10527   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10528 
10529   ins_encode %{
10530     __ add(as_Register($dst$$reg),
10531            as_Register($src1$$reg),
10532            as_Register($src2$$reg), ext::sxtw);
10533   %}
10534 
10535   ins_pipe(ialu_reg_reg);
10536 %}
10537 
10538 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10539   match(Set dst (AddP src1 (LShiftL src2 scale)));
10540 
10541   ins_cost(1.9 * INSN_COST);
10542   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10543 
10544   ins_encode %{
10545     __ lea(as_Register($dst$$reg),
10546            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10547                    Address::lsl($scale$$constant)));
10548   %}
10549 
10550   ins_pipe(ialu_reg_reg_shift);
10551 %}
10552 
10553 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10554   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10555 
10556   ins_cost(1.9 * INSN_COST);
10557   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10558 
10559   ins_encode %{
10560     __ lea(as_Register($dst$$reg),
10561            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10562                    Address::sxtw($scale$$constant)));
10563   %}
10564 
10565   ins_pipe(ialu_reg_reg_shift);
10566 %}
10567 
10568 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10569   match(Set dst (LShiftL (ConvI2L src) scale));
10570 
10571   ins_cost(INSN_COST);
10572   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10573 
10574   ins_encode %{
10575     __ sbfiz(as_Register($dst$$reg),
10576           as_Register($src$$reg),
10577           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10578   %}
10579 
10580   ins_pipe(ialu_reg_shift);
10581 %}
10582 
10583 // Pointer Immediate Addition
10584 // n.b. this needs to be more expensive than using an indirect memory
10585 // operand
10586 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10587   match(Set dst (AddP src1 src2));
10588 
10589   ins_cost(INSN_COST);
10590   format %{ "add $dst, $src1, $src2\t# ptr" %}
10591 
10592   // use opcode to indicate that this is an add not a sub
10593   opcode(0x0);
10594 
10595   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10596 
10597   ins_pipe(ialu_reg_imm);
10598 %}
10599 
10600 // Long Addition
10601 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10602 
10603   match(Set dst (AddL src1 src2));
10604 
10605   ins_cost(INSN_COST);
10606   format %{ "add  $dst, $src1, $src2" %}
10607 
10608   ins_encode %{
10609     __ add(as_Register($dst$$reg),
10610            as_Register($src1$$reg),
10611            as_Register($src2$$reg));
10612   %}
10613 
10614   ins_pipe(ialu_reg_reg);
10615 %}
10616 
10617 // No constant pool entries requiredLong Immediate Addition.
10618 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10619   match(Set dst (AddL src1 src2));
10620 
10621   ins_cost(INSN_COST);
10622   format %{ "add $dst, $src1, $src2" %}
10623 
10624   // use opcode to indicate that this is an add not a sub
10625   opcode(0x0);
10626 
10627   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10628 
10629   ins_pipe(ialu_reg_imm);
10630 %}
10631 
10632 // Integer Subtraction
10633 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10634   match(Set dst (SubI src1 src2));
10635 
10636   ins_cost(INSN_COST);
10637   format %{ "subw  $dst, $src1, $src2" %}
10638 
10639   ins_encode %{
10640     __ subw(as_Register($dst$$reg),
10641             as_Register($src1$$reg),
10642             as_Register($src2$$reg));
10643   %}
10644 
10645   ins_pipe(ialu_reg_reg);
10646 %}
10647 
10648 // Immediate Subtraction
10649 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10650   match(Set dst (SubI src1 src2));
10651 
10652   ins_cost(INSN_COST);
10653   format %{ "subw $dst, $src1, $src2" %}
10654 
10655   // use opcode to indicate that this is a sub not an add
10656   opcode(0x1);
10657 
10658   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10659 
10660   ins_pipe(ialu_reg_imm);
10661 %}
10662 
10663 // Long Subtraction
10664 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10665 
10666   match(Set dst (SubL src1 src2));
10667 
10668   ins_cost(INSN_COST);
10669   format %{ "sub  $dst, $src1, $src2" %}
10670 
10671   ins_encode %{
10672     __ sub(as_Register($dst$$reg),
10673            as_Register($src1$$reg),
10674            as_Register($src2$$reg));
10675   %}
10676 
10677   ins_pipe(ialu_reg_reg);
10678 %}
10679 
10680 // No constant pool entries requiredLong Immediate Subtraction.
10681 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10682   match(Set dst (SubL src1 src2));
10683 
10684   ins_cost(INSN_COST);
10685   format %{ "sub$dst, $src1, $src2" %}
10686 
10687   // use opcode to indicate that this is a sub not an add
10688   opcode(0x1);
10689 
10690   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10691 
10692   ins_pipe(ialu_reg_imm);
10693 %}
10694 
10695 // Integer Negation (special case for sub)
10696 
10697 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10698   match(Set dst (SubI zero src));
10699 
10700   ins_cost(INSN_COST);
10701   format %{ "negw $dst, $src\t# int" %}
10702 
10703   ins_encode %{
10704     __ negw(as_Register($dst$$reg),
10705             as_Register($src$$reg));
10706   %}
10707 
10708   ins_pipe(ialu_reg);
10709 %}
10710 
10711 // Long Negation
10712 
10713 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10714   match(Set dst (SubL zero src));
10715 
10716   ins_cost(INSN_COST);
10717   format %{ "neg $dst, $src\t# long" %}
10718 
10719   ins_encode %{
10720     __ neg(as_Register($dst$$reg),
10721            as_Register($src$$reg));
10722   %}
10723 
10724   ins_pipe(ialu_reg);
10725 %}
10726 
10727 // Integer Multiply
10728 
10729 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10730   match(Set dst (MulI src1 src2));
10731 
10732   ins_cost(INSN_COST * 3);
10733   format %{ "mulw  $dst, $src1, $src2" %}
10734 
10735   ins_encode %{
10736     __ mulw(as_Register($dst$$reg),
10737             as_Register($src1$$reg),
10738             as_Register($src2$$reg));
10739   %}
10740 
10741   ins_pipe(imul_reg_reg);
10742 %}
10743 
10744 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10745   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10746 
10747   ins_cost(INSN_COST * 3);
10748   format %{ "smull  $dst, $src1, $src2" %}
10749 
10750   ins_encode %{
10751     __ smull(as_Register($dst$$reg),
10752              as_Register($src1$$reg),
10753              as_Register($src2$$reg));
10754   %}
10755 
10756   ins_pipe(imul_reg_reg);
10757 %}
10758 
10759 // Long Multiply
10760 
10761 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10762   match(Set dst (MulL src1 src2));
10763 
10764   ins_cost(INSN_COST * 5);
10765   format %{ "mul  $dst, $src1, $src2" %}
10766 
10767   ins_encode %{
10768     __ mul(as_Register($dst$$reg),
10769            as_Register($src1$$reg),
10770            as_Register($src2$$reg));
10771   %}
10772 
10773   ins_pipe(lmul_reg_reg);
10774 %}
10775 
10776 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10777 %{
10778   match(Set dst (MulHiL src1 src2));
10779 
10780   ins_cost(INSN_COST * 7);
10781   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10782 
10783   ins_encode %{
10784     __ smulh(as_Register($dst$$reg),
10785              as_Register($src1$$reg),
10786              as_Register($src2$$reg));
10787   %}
10788 
10789   ins_pipe(lmul_reg_reg);
10790 %}
10791 
10792 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10793 %{
10794   match(Set dst (UMulHiL src1 src2));
10795 
10796   ins_cost(INSN_COST * 7);
10797   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10798 
10799   ins_encode %{
10800     __ umulh(as_Register($dst$$reg),
10801              as_Register($src1$$reg),
10802              as_Register($src2$$reg));
10803   %}
10804 
10805   ins_pipe(lmul_reg_reg);
10806 %}
10807 
10808 // Combined Integer Multiply & Add/Sub
10809 
10810 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10811   match(Set dst (AddI src3 (MulI src1 src2)));
10812 
10813   ins_cost(INSN_COST * 3);
10814   format %{ "madd  $dst, $src1, $src2, $src3" %}
10815 
10816   ins_encode %{
10817     __ maddw(as_Register($dst$$reg),
10818              as_Register($src1$$reg),
10819              as_Register($src2$$reg),
10820              as_Register($src3$$reg));
10821   %}
10822 
10823   ins_pipe(imac_reg_reg);
10824 %}
10825 
10826 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10827   match(Set dst (SubI src3 (MulI src1 src2)));
10828 
10829   ins_cost(INSN_COST * 3);
10830   format %{ "msub  $dst, $src1, $src2, $src3" %}
10831 
10832   ins_encode %{
10833     __ msubw(as_Register($dst$$reg),
10834              as_Register($src1$$reg),
10835              as_Register($src2$$reg),
10836              as_Register($src3$$reg));
10837   %}
10838 
10839   ins_pipe(imac_reg_reg);
10840 %}
10841 
10842 // Combined Integer Multiply & Neg
10843 
10844 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10845   match(Set dst (MulI (SubI zero src1) src2));
10846 
10847   ins_cost(INSN_COST * 3);
10848   format %{ "mneg  $dst, $src1, $src2" %}
10849 
10850   ins_encode %{
10851     __ mnegw(as_Register($dst$$reg),
10852              as_Register($src1$$reg),
10853              as_Register($src2$$reg));
10854   %}
10855 
10856   ins_pipe(imac_reg_reg);
10857 %}
10858 
10859 // Combined Long Multiply & Add/Sub
10860 
10861 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10862   match(Set dst (AddL src3 (MulL src1 src2)));
10863 
10864   ins_cost(INSN_COST * 5);
10865   format %{ "madd  $dst, $src1, $src2, $src3" %}
10866 
10867   ins_encode %{
10868     __ madd(as_Register($dst$$reg),
10869             as_Register($src1$$reg),
10870             as_Register($src2$$reg),
10871             as_Register($src3$$reg));
10872   %}
10873 
10874   ins_pipe(lmac_reg_reg);
10875 %}
10876 
10877 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10878   match(Set dst (SubL src3 (MulL src1 src2)));
10879 
10880   ins_cost(INSN_COST * 5);
10881   format %{ "msub  $dst, $src1, $src2, $src3" %}
10882 
10883   ins_encode %{
10884     __ msub(as_Register($dst$$reg),
10885             as_Register($src1$$reg),
10886             as_Register($src2$$reg),
10887             as_Register($src3$$reg));
10888   %}
10889 
10890   ins_pipe(lmac_reg_reg);
10891 %}
10892 
10893 // Combined Long Multiply & Neg
10894 
10895 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10896   match(Set dst (MulL (SubL zero src1) src2));
10897 
10898   ins_cost(INSN_COST * 5);
10899   format %{ "mneg  $dst, $src1, $src2" %}
10900 
10901   ins_encode %{
10902     __ mneg(as_Register($dst$$reg),
10903             as_Register($src1$$reg),
10904             as_Register($src2$$reg));
10905   %}
10906 
10907   ins_pipe(lmac_reg_reg);
10908 %}
10909 
10910 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10911 
10912 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10913   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10914 
10915   ins_cost(INSN_COST * 3);
10916   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10917 
10918   ins_encode %{
10919     __ smaddl(as_Register($dst$$reg),
10920               as_Register($src1$$reg),
10921               as_Register($src2$$reg),
10922               as_Register($src3$$reg));
10923   %}
10924 
10925   ins_pipe(imac_reg_reg);
10926 %}
10927 
10928 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10929   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10930 
10931   ins_cost(INSN_COST * 3);
10932   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10933 
10934   ins_encode %{
10935     __ smsubl(as_Register($dst$$reg),
10936               as_Register($src1$$reg),
10937               as_Register($src2$$reg),
10938               as_Register($src3$$reg));
10939   %}
10940 
10941   ins_pipe(imac_reg_reg);
10942 %}
10943 
10944 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10945   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10946 
10947   ins_cost(INSN_COST * 3);
10948   format %{ "smnegl  $dst, $src1, $src2" %}
10949 
10950   ins_encode %{
10951     __ smnegl(as_Register($dst$$reg),
10952               as_Register($src1$$reg),
10953               as_Register($src2$$reg));
10954   %}
10955 
10956   ins_pipe(imac_reg_reg);
10957 %}
10958 
10959 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10960 
10961 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10962   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10963 
10964   ins_cost(INSN_COST * 5);
10965   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10966             "maddw $dst, $src3, $src4, rscratch1" %}
10967 
10968   ins_encode %{
10969     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10970     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10971 
10972   ins_pipe(imac_reg_reg);
10973 %}
10974 
10975 // Integer Divide
10976 
10977 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10978   match(Set dst (DivI src1 src2));
10979 
10980   ins_cost(INSN_COST * 19);
10981   format %{ "sdivw  $dst, $src1, $src2" %}
10982 
10983   ins_encode(aarch64_enc_divw(dst, src1, src2));
10984   ins_pipe(idiv_reg_reg);
10985 %}
10986 
10987 // Long Divide
10988 
10989 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10990   match(Set dst (DivL src1 src2));
10991 
10992   ins_cost(INSN_COST * 35);
10993   format %{ "sdiv   $dst, $src1, $src2" %}
10994 
10995   ins_encode(aarch64_enc_div(dst, src1, src2));
10996   ins_pipe(ldiv_reg_reg);
10997 %}
10998 
10999 // Integer Remainder
11000 
11001 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11002   match(Set dst (ModI src1 src2));
11003 
11004   ins_cost(INSN_COST * 22);
11005   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
11006             "msubw  $dst, rscratch1, $src2, $src1" %}
11007 
11008   ins_encode(aarch64_enc_modw(dst, src1, src2));
11009   ins_pipe(idiv_reg_reg);
11010 %}
11011 
11012 // Long Remainder
11013 
11014 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11015   match(Set dst (ModL src1 src2));
11016 
11017   ins_cost(INSN_COST * 38);
11018   format %{ "sdiv   rscratch1, $src1, $src2\n"
11019             "msub   $dst, rscratch1, $src2, $src1" %}
11020 
11021   ins_encode(aarch64_enc_mod(dst, src1, src2));
11022   ins_pipe(ldiv_reg_reg);
11023 %}
11024 
11025 // Unsigned Integer Divide
11026 
11027 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11028   match(Set dst (UDivI src1 src2));
11029 
11030   ins_cost(INSN_COST * 19);
11031   format %{ "udivw  $dst, $src1, $src2" %}
11032 
11033   ins_encode %{
11034     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
11035   %}
11036 
11037   ins_pipe(idiv_reg_reg);
11038 %}
11039 
11040 //  Unsigned Long Divide
11041 
11042 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11043   match(Set dst (UDivL src1 src2));
11044 
11045   ins_cost(INSN_COST * 35);
11046   format %{ "udiv   $dst, $src1, $src2" %}
11047 
11048   ins_encode %{
11049     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
11050   %}
11051 
11052   ins_pipe(ldiv_reg_reg);
11053 %}
11054 
11055 // Unsigned Integer Remainder
11056 
11057 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11058   match(Set dst (UModI src1 src2));
11059 
11060   ins_cost(INSN_COST * 22);
11061   format %{ "udivw  rscratch1, $src1, $src2\n\t"
11062             "msubw  $dst, rscratch1, $src2, $src1" %}
11063 
11064   ins_encode %{
11065     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11066     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11067   %}
11068 
11069   ins_pipe(idiv_reg_reg);
11070 %}
11071 
11072 // Unsigned Long Remainder
11073 
11074 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11075   match(Set dst (UModL src1 src2));
11076 
11077   ins_cost(INSN_COST * 38);
11078   format %{ "udiv   rscratch1, $src1, $src2\n"
11079             "msub   $dst, rscratch1, $src2, $src1" %}
11080 
11081   ins_encode %{
11082     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11083     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11084   %}
11085 
11086   ins_pipe(ldiv_reg_reg);
11087 %}
11088 
11089 // Integer Shifts
11090 
11091 // Shift Left Register
11092 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11093   match(Set dst (LShiftI src1 src2));
11094 
11095   ins_cost(INSN_COST * 2);
11096   format %{ "lslvw  $dst, $src1, $src2" %}
11097 
11098   ins_encode %{
11099     __ lslvw(as_Register($dst$$reg),
11100              as_Register($src1$$reg),
11101              as_Register($src2$$reg));
11102   %}
11103 
11104   ins_pipe(ialu_reg_reg_vshift);
11105 %}
11106 
11107 // Shift Left Immediate
11108 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11109   match(Set dst (LShiftI src1 src2));
11110 
11111   ins_cost(INSN_COST);
11112   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11113 
11114   ins_encode %{
11115     __ lslw(as_Register($dst$$reg),
11116             as_Register($src1$$reg),
11117             $src2$$constant & 0x1f);
11118   %}
11119 
11120   ins_pipe(ialu_reg_shift);
11121 %}
11122 
11123 // Shift Right Logical Register
11124 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11125   match(Set dst (URShiftI src1 src2));
11126 
11127   ins_cost(INSN_COST * 2);
11128   format %{ "lsrvw  $dst, $src1, $src2" %}
11129 
11130   ins_encode %{
11131     __ lsrvw(as_Register($dst$$reg),
11132              as_Register($src1$$reg),
11133              as_Register($src2$$reg));
11134   %}
11135 
11136   ins_pipe(ialu_reg_reg_vshift);
11137 %}
11138 
11139 // Shift Right Logical Immediate
11140 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11141   match(Set dst (URShiftI src1 src2));
11142 
11143   ins_cost(INSN_COST);
11144   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11145 
11146   ins_encode %{
11147     __ lsrw(as_Register($dst$$reg),
11148             as_Register($src1$$reg),
11149             $src2$$constant & 0x1f);
11150   %}
11151 
11152   ins_pipe(ialu_reg_shift);
11153 %}
11154 
11155 // Shift Right Arithmetic Register
11156 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11157   match(Set dst (RShiftI src1 src2));
11158 
11159   ins_cost(INSN_COST * 2);
11160   format %{ "asrvw  $dst, $src1, $src2" %}
11161 
11162   ins_encode %{
11163     __ asrvw(as_Register($dst$$reg),
11164              as_Register($src1$$reg),
11165              as_Register($src2$$reg));
11166   %}
11167 
11168   ins_pipe(ialu_reg_reg_vshift);
11169 %}
11170 
11171 // Shift Right Arithmetic Immediate
11172 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11173   match(Set dst (RShiftI src1 src2));
11174 
11175   ins_cost(INSN_COST);
11176   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11177 
11178   ins_encode %{
11179     __ asrw(as_Register($dst$$reg),
11180             as_Register($src1$$reg),
11181             $src2$$constant & 0x1f);
11182   %}
11183 
11184   ins_pipe(ialu_reg_shift);
11185 %}
11186 
11187 // Combined Int Mask and Right Shift (using UBFM)
11188 // TODO
11189 
11190 // Long Shifts
11191 
11192 // Shift Left Register
11193 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11194   match(Set dst (LShiftL src1 src2));
11195 
11196   ins_cost(INSN_COST * 2);
11197   format %{ "lslv  $dst, $src1, $src2" %}
11198 
11199   ins_encode %{
11200     __ lslv(as_Register($dst$$reg),
11201             as_Register($src1$$reg),
11202             as_Register($src2$$reg));
11203   %}
11204 
11205   ins_pipe(ialu_reg_reg_vshift);
11206 %}
11207 
11208 // Shift Left Immediate
11209 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11210   match(Set dst (LShiftL src1 src2));
11211 
11212   ins_cost(INSN_COST);
11213   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11214 
11215   ins_encode %{
11216     __ lsl(as_Register($dst$$reg),
11217             as_Register($src1$$reg),
11218             $src2$$constant & 0x3f);
11219   %}
11220 
11221   ins_pipe(ialu_reg_shift);
11222 %}
11223 
11224 // Shift Right Logical Register
11225 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11226   match(Set dst (URShiftL src1 src2));
11227 
11228   ins_cost(INSN_COST * 2);
11229   format %{ "lsrv  $dst, $src1, $src2" %}
11230 
11231   ins_encode %{
11232     __ lsrv(as_Register($dst$$reg),
11233             as_Register($src1$$reg),
11234             as_Register($src2$$reg));
11235   %}
11236 
11237   ins_pipe(ialu_reg_reg_vshift);
11238 %}
11239 
11240 // Shift Right Logical Immediate
11241 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11242   match(Set dst (URShiftL src1 src2));
11243 
11244   ins_cost(INSN_COST);
11245   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11246 
11247   ins_encode %{
11248     __ lsr(as_Register($dst$$reg),
11249            as_Register($src1$$reg),
11250            $src2$$constant & 0x3f);
11251   %}
11252 
11253   ins_pipe(ialu_reg_shift);
11254 %}
11255 
11256 // A special-case pattern for card table stores.
11257 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11258   match(Set dst (URShiftL (CastP2X src1) src2));
11259 
11260   ins_cost(INSN_COST);
11261   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11262 
11263   ins_encode %{
11264     __ lsr(as_Register($dst$$reg),
11265            as_Register($src1$$reg),
11266            $src2$$constant & 0x3f);
11267   %}
11268 
11269   ins_pipe(ialu_reg_shift);
11270 %}
11271 
11272 // Shift Right Arithmetic Register
11273 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11274   match(Set dst (RShiftL src1 src2));
11275 
11276   ins_cost(INSN_COST * 2);
11277   format %{ "asrv  $dst, $src1, $src2" %}
11278 
11279   ins_encode %{
11280     __ asrv(as_Register($dst$$reg),
11281             as_Register($src1$$reg),
11282             as_Register($src2$$reg));
11283   %}
11284 
11285   ins_pipe(ialu_reg_reg_vshift);
11286 %}
11287 
11288 // Shift Right Arithmetic Immediate
11289 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11290   match(Set dst (RShiftL src1 src2));
11291 
11292   ins_cost(INSN_COST);
11293   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11294 
11295   ins_encode %{
11296     __ asr(as_Register($dst$$reg),
11297            as_Register($src1$$reg),
11298            $src2$$constant & 0x3f);
11299   %}
11300 
11301   ins_pipe(ialu_reg_shift);
11302 %}
11303 
11304 // BEGIN This section of the file is automatically generated. Do not edit --------------
11305 // This section is generated from aarch64_ad.m4
11306 
11307 // This pattern is automatically generated from aarch64_ad.m4
11308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11309 instruct regL_not_reg(iRegLNoSp dst,
11310                          iRegL src1, immL_M1 m1,
11311                          rFlagsReg cr) %{
11312   match(Set dst (XorL src1 m1));
11313   ins_cost(INSN_COST);
11314   format %{ "eon  $dst, $src1, zr" %}
11315 
11316   ins_encode %{
11317     __ eon(as_Register($dst$$reg),
11318               as_Register($src1$$reg),
11319               zr,
11320               Assembler::LSL, 0);
11321   %}
11322 
11323   ins_pipe(ialu_reg);
11324 %}
11325 
11326 // This pattern is automatically generated from aarch64_ad.m4
11327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11328 instruct regI_not_reg(iRegINoSp dst,
11329                          iRegIorL2I src1, immI_M1 m1,
11330                          rFlagsReg cr) %{
11331   match(Set dst (XorI src1 m1));
11332   ins_cost(INSN_COST);
11333   format %{ "eonw  $dst, $src1, zr" %}
11334 
11335   ins_encode %{
11336     __ eonw(as_Register($dst$$reg),
11337               as_Register($src1$$reg),
11338               zr,
11339               Assembler::LSL, 0);
11340   %}
11341 
11342   ins_pipe(ialu_reg);
11343 %}
11344 
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11348                               immI0 zero, iRegIorL2I src1, immI src2) %{
11349   match(Set dst (SubI zero (URShiftI src1 src2)));
11350 
11351   ins_cost(1.9 * INSN_COST);
11352   format %{ "negw  $dst, $src1, LSR $src2" %}
11353 
11354   ins_encode %{
11355     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11356             Assembler::LSR, $src2$$constant & 0x1f);
11357   %}
11358 
11359   ins_pipe(ialu_reg_shift);
11360 %}
11361 
11362 // This pattern is automatically generated from aarch64_ad.m4
11363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11364 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11365                               immI0 zero, iRegIorL2I src1, immI src2) %{
11366   match(Set dst (SubI zero (RShiftI src1 src2)));
11367 
11368   ins_cost(1.9 * INSN_COST);
11369   format %{ "negw  $dst, $src1, ASR $src2" %}
11370 
11371   ins_encode %{
11372     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11373             Assembler::ASR, $src2$$constant & 0x1f);
11374   %}
11375 
11376   ins_pipe(ialu_reg_shift);
11377 %}
11378 
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11382                               immI0 zero, iRegIorL2I src1, immI src2) %{
11383   match(Set dst (SubI zero (LShiftI src1 src2)));
11384 
11385   ins_cost(1.9 * INSN_COST);
11386   format %{ "negw  $dst, $src1, LSL $src2" %}
11387 
11388   ins_encode %{
11389     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11390             Assembler::LSL, $src2$$constant & 0x1f);
11391   %}
11392 
11393   ins_pipe(ialu_reg_shift);
11394 %}
11395 
11396 // This pattern is automatically generated from aarch64_ad.m4
11397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11398 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11399                               immL0 zero, iRegL src1, immI src2) %{
11400   match(Set dst (SubL zero (URShiftL src1 src2)));
11401 
11402   ins_cost(1.9 * INSN_COST);
11403   format %{ "neg  $dst, $src1, LSR $src2" %}
11404 
11405   ins_encode %{
11406     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11407             Assembler::LSR, $src2$$constant & 0x3f);
11408   %}
11409 
11410   ins_pipe(ialu_reg_shift);
11411 %}
11412 
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11416                               immL0 zero, iRegL src1, immI src2) %{
11417   match(Set dst (SubL zero (RShiftL src1 src2)));
11418 
11419   ins_cost(1.9 * INSN_COST);
11420   format %{ "neg  $dst, $src1, ASR $src2" %}
11421 
11422   ins_encode %{
11423     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11424             Assembler::ASR, $src2$$constant & 0x3f);
11425   %}
11426 
11427   ins_pipe(ialu_reg_shift);
11428 %}
11429 
11430 // This pattern is automatically generated from aarch64_ad.m4
11431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11432 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11433                               immL0 zero, iRegL src1, immI src2) %{
11434   match(Set dst (SubL zero (LShiftL src1 src2)));
11435 
11436   ins_cost(1.9 * INSN_COST);
11437   format %{ "neg  $dst, $src1, LSL $src2" %}
11438 
11439   ins_encode %{
11440     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11441             Assembler::LSL, $src2$$constant & 0x3f);
11442   %}
11443 
11444   ins_pipe(ialu_reg_shift);
11445 %}
11446 
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449 instruct AndI_reg_not_reg(iRegINoSp dst,
11450                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11451   match(Set dst (AndI src1 (XorI src2 m1)));
11452   ins_cost(INSN_COST);
11453   format %{ "bicw  $dst, $src1, $src2" %}
11454 
11455   ins_encode %{
11456     __ bicw(as_Register($dst$$reg),
11457               as_Register($src1$$reg),
11458               as_Register($src2$$reg),
11459               Assembler::LSL, 0);
11460   %}
11461 
11462   ins_pipe(ialu_reg_reg);
11463 %}
11464 
11465 // This pattern is automatically generated from aarch64_ad.m4
11466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11467 instruct AndL_reg_not_reg(iRegLNoSp dst,
11468                          iRegL src1, iRegL src2, immL_M1 m1) %{
11469   match(Set dst (AndL src1 (XorL src2 m1)));
11470   ins_cost(INSN_COST);
11471   format %{ "bic  $dst, $src1, $src2" %}
11472 
11473   ins_encode %{
11474     __ bic(as_Register($dst$$reg),
11475               as_Register($src1$$reg),
11476               as_Register($src2$$reg),
11477               Assembler::LSL, 0);
11478   %}
11479 
11480   ins_pipe(ialu_reg_reg);
11481 %}
11482 
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 instruct OrI_reg_not_reg(iRegINoSp dst,
11486                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11487   match(Set dst (OrI src1 (XorI src2 m1)));
11488   ins_cost(INSN_COST);
11489   format %{ "ornw  $dst, $src1, $src2" %}
11490 
11491   ins_encode %{
11492     __ ornw(as_Register($dst$$reg),
11493               as_Register($src1$$reg),
11494               as_Register($src2$$reg),
11495               Assembler::LSL, 0);
11496   %}
11497 
11498   ins_pipe(ialu_reg_reg);
11499 %}
11500 
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503 instruct OrL_reg_not_reg(iRegLNoSp dst,
11504                          iRegL src1, iRegL src2, immL_M1 m1) %{
11505   match(Set dst (OrL src1 (XorL src2 m1)));
11506   ins_cost(INSN_COST);
11507   format %{ "orn  $dst, $src1, $src2" %}
11508 
11509   ins_encode %{
11510     __ orn(as_Register($dst$$reg),
11511               as_Register($src1$$reg),
11512               as_Register($src2$$reg),
11513               Assembler::LSL, 0);
11514   %}
11515 
11516   ins_pipe(ialu_reg_reg);
11517 %}
11518 
11519 // This pattern is automatically generated from aarch64_ad.m4
11520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11521 instruct XorI_reg_not_reg(iRegINoSp dst,
11522                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11523   match(Set dst (XorI m1 (XorI src2 src1)));
11524   ins_cost(INSN_COST);
11525   format %{ "eonw  $dst, $src1, $src2" %}
11526 
11527   ins_encode %{
11528     __ eonw(as_Register($dst$$reg),
11529               as_Register($src1$$reg),
11530               as_Register($src2$$reg),
11531               Assembler::LSL, 0);
11532   %}
11533 
11534   ins_pipe(ialu_reg_reg);
11535 %}
11536 
11537 // This pattern is automatically generated from aarch64_ad.m4
11538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11539 instruct XorL_reg_not_reg(iRegLNoSp dst,
11540                          iRegL src1, iRegL src2, immL_M1 m1) %{
11541   match(Set dst (XorL m1 (XorL src2 src1)));
11542   ins_cost(INSN_COST);
11543   format %{ "eon  $dst, $src1, $src2" %}
11544 
11545   ins_encode %{
11546     __ eon(as_Register($dst$$reg),
11547               as_Register($src1$$reg),
11548               as_Register($src2$$reg),
11549               Assembler::LSL, 0);
11550   %}
11551 
11552   ins_pipe(ialu_reg_reg);
11553 %}
11554 
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 // val & (-1 ^ (val >>> shift)) ==> bicw
11558 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11559                          iRegIorL2I src1, iRegIorL2I src2,
11560                          immI src3, immI_M1 src4) %{
11561   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11562   ins_cost(1.9 * INSN_COST);
11563   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11564 
11565   ins_encode %{
11566     __ bicw(as_Register($dst$$reg),
11567               as_Register($src1$$reg),
11568               as_Register($src2$$reg),
11569               Assembler::LSR,
11570               $src3$$constant & 0x1f);
11571   %}
11572 
11573   ins_pipe(ialu_reg_reg_shift);
11574 %}
11575 
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 // val & (-1 ^ (val >>> shift)) ==> bic
11579 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11580                          iRegL src1, iRegL src2,
11581                          immI src3, immL_M1 src4) %{
11582   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11583   ins_cost(1.9 * INSN_COST);
11584   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11585 
11586   ins_encode %{
11587     __ bic(as_Register($dst$$reg),
11588               as_Register($src1$$reg),
11589               as_Register($src2$$reg),
11590               Assembler::LSR,
11591               $src3$$constant & 0x3f);
11592   %}
11593 
11594   ins_pipe(ialu_reg_reg_shift);
11595 %}
11596 
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 // val & (-1 ^ (val >> shift)) ==> bicw
11600 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11601                          iRegIorL2I src1, iRegIorL2I src2,
11602                          immI src3, immI_M1 src4) %{
11603   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11604   ins_cost(1.9 * INSN_COST);
11605   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11606 
11607   ins_encode %{
11608     __ bicw(as_Register($dst$$reg),
11609               as_Register($src1$$reg),
11610               as_Register($src2$$reg),
11611               Assembler::ASR,
11612               $src3$$constant & 0x1f);
11613   %}
11614 
11615   ins_pipe(ialu_reg_reg_shift);
11616 %}
11617 
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620 // val & (-1 ^ (val >> shift)) ==> bic
11621 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11622                          iRegL src1, iRegL src2,
11623                          immI src3, immL_M1 src4) %{
11624   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11625   ins_cost(1.9 * INSN_COST);
11626   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11627 
11628   ins_encode %{
11629     __ bic(as_Register($dst$$reg),
11630               as_Register($src1$$reg),
11631               as_Register($src2$$reg),
11632               Assembler::ASR,
11633               $src3$$constant & 0x3f);
11634   %}
11635 
11636   ins_pipe(ialu_reg_reg_shift);
11637 %}
11638 
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641 // val & (-1 ^ (val ror shift)) ==> bicw
11642 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11643                          iRegIorL2I src1, iRegIorL2I src2,
11644                          immI src3, immI_M1 src4) %{
11645   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11646   ins_cost(1.9 * INSN_COST);
11647   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11648 
11649   ins_encode %{
11650     __ bicw(as_Register($dst$$reg),
11651               as_Register($src1$$reg),
11652               as_Register($src2$$reg),
11653               Assembler::ROR,
11654               $src3$$constant & 0x1f);
11655   %}
11656 
11657   ins_pipe(ialu_reg_reg_shift);
11658 %}
11659 
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662 // val & (-1 ^ (val ror shift)) ==> bic
11663 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11664                          iRegL src1, iRegL src2,
11665                          immI src3, immL_M1 src4) %{
11666   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11667   ins_cost(1.9 * INSN_COST);
11668   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11669 
11670   ins_encode %{
11671     __ bic(as_Register($dst$$reg),
11672               as_Register($src1$$reg),
11673               as_Register($src2$$reg),
11674               Assembler::ROR,
11675               $src3$$constant & 0x3f);
11676   %}
11677 
11678   ins_pipe(ialu_reg_reg_shift);
11679 %}
11680 
11681 // This pattern is automatically generated from aarch64_ad.m4
11682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11683 // val & (-1 ^ (val << shift)) ==> bicw
11684 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11685                          iRegIorL2I src1, iRegIorL2I src2,
11686                          immI src3, immI_M1 src4) %{
11687   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11688   ins_cost(1.9 * INSN_COST);
11689   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11690 
11691   ins_encode %{
11692     __ bicw(as_Register($dst$$reg),
11693               as_Register($src1$$reg),
11694               as_Register($src2$$reg),
11695               Assembler::LSL,
11696               $src3$$constant & 0x1f);
11697   %}
11698 
11699   ins_pipe(ialu_reg_reg_shift);
11700 %}
11701 
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 // val & (-1 ^ (val << shift)) ==> bic
11705 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11706                          iRegL src1, iRegL src2,
11707                          immI src3, immL_M1 src4) %{
11708   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11709   ins_cost(1.9 * INSN_COST);
11710   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11711 
11712   ins_encode %{
11713     __ bic(as_Register($dst$$reg),
11714               as_Register($src1$$reg),
11715               as_Register($src2$$reg),
11716               Assembler::LSL,
11717               $src3$$constant & 0x3f);
11718   %}
11719 
11720   ins_pipe(ialu_reg_reg_shift);
11721 %}
11722 
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11726 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11727                          iRegIorL2I src1, iRegIorL2I src2,
11728                          immI src3, immI_M1 src4) %{
11729   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11730   ins_cost(1.9 * INSN_COST);
11731   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11732 
11733   ins_encode %{
11734     __ eonw(as_Register($dst$$reg),
11735               as_Register($src1$$reg),
11736               as_Register($src2$$reg),
11737               Assembler::LSR,
11738               $src3$$constant & 0x1f);
11739   %}
11740 
11741   ins_pipe(ialu_reg_reg_shift);
11742 %}
11743 
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 // val ^ (-1 ^ (val >>> shift)) ==> eon
11747 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11748                          iRegL src1, iRegL src2,
11749                          immI src3, immL_M1 src4) %{
11750   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11751   ins_cost(1.9 * INSN_COST);
11752   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11753 
11754   ins_encode %{
11755     __ eon(as_Register($dst$$reg),
11756               as_Register($src1$$reg),
11757               as_Register($src2$$reg),
11758               Assembler::LSR,
11759               $src3$$constant & 0x3f);
11760   %}
11761 
11762   ins_pipe(ialu_reg_reg_shift);
11763 %}
11764 
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 // val ^ (-1 ^ (val >> shift)) ==> eonw
11768 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11769                          iRegIorL2I src1, iRegIorL2I src2,
11770                          immI src3, immI_M1 src4) %{
11771   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11772   ins_cost(1.9 * INSN_COST);
11773   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11774 
11775   ins_encode %{
11776     __ eonw(as_Register($dst$$reg),
11777               as_Register($src1$$reg),
11778               as_Register($src2$$reg),
11779               Assembler::ASR,
11780               $src3$$constant & 0x1f);
11781   %}
11782 
11783   ins_pipe(ialu_reg_reg_shift);
11784 %}
11785 
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 // val ^ (-1 ^ (val >> shift)) ==> eon
11789 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11790                          iRegL src1, iRegL src2,
11791                          immI src3, immL_M1 src4) %{
11792   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11793   ins_cost(1.9 * INSN_COST);
11794   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11795 
11796   ins_encode %{
11797     __ eon(as_Register($dst$$reg),
11798               as_Register($src1$$reg),
11799               as_Register($src2$$reg),
11800               Assembler::ASR,
11801               $src3$$constant & 0x3f);
11802   %}
11803 
11804   ins_pipe(ialu_reg_reg_shift);
11805 %}
11806 
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 // val ^ (-1 ^ (val ror shift)) ==> eonw
11810 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11811                          iRegIorL2I src1, iRegIorL2I src2,
11812                          immI src3, immI_M1 src4) %{
11813   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11814   ins_cost(1.9 * INSN_COST);
11815   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11816 
11817   ins_encode %{
11818     __ eonw(as_Register($dst$$reg),
11819               as_Register($src1$$reg),
11820               as_Register($src2$$reg),
11821               Assembler::ROR,
11822               $src3$$constant & 0x1f);
11823   %}
11824 
11825   ins_pipe(ialu_reg_reg_shift);
11826 %}
11827 
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 // val ^ (-1 ^ (val ror shift)) ==> eon
11831 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11832                          iRegL src1, iRegL src2,
11833                          immI src3, immL_M1 src4) %{
11834   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11835   ins_cost(1.9 * INSN_COST);
11836   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11837 
11838   ins_encode %{
11839     __ eon(as_Register($dst$$reg),
11840               as_Register($src1$$reg),
11841               as_Register($src2$$reg),
11842               Assembler::ROR,
11843               $src3$$constant & 0x3f);
11844   %}
11845 
11846   ins_pipe(ialu_reg_reg_shift);
11847 %}
11848 
11849 // This pattern is automatically generated from aarch64_ad.m4
11850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11851 // val ^ (-1 ^ (val << shift)) ==> eonw
11852 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11853                          iRegIorL2I src1, iRegIorL2I src2,
11854                          immI src3, immI_M1 src4) %{
11855   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11856   ins_cost(1.9 * INSN_COST);
11857   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11858 
11859   ins_encode %{
11860     __ eonw(as_Register($dst$$reg),
11861               as_Register($src1$$reg),
11862               as_Register($src2$$reg),
11863               Assembler::LSL,
11864               $src3$$constant & 0x1f);
11865   %}
11866 
11867   ins_pipe(ialu_reg_reg_shift);
11868 %}
11869 
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 // val ^ (-1 ^ (val << shift)) ==> eon
11873 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11874                          iRegL src1, iRegL src2,
11875                          immI src3, immL_M1 src4) %{
11876   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11877   ins_cost(1.9 * INSN_COST);
11878   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11879 
11880   ins_encode %{
11881     __ eon(as_Register($dst$$reg),
11882               as_Register($src1$$reg),
11883               as_Register($src2$$reg),
11884               Assembler::LSL,
11885               $src3$$constant & 0x3f);
11886   %}
11887 
11888   ins_pipe(ialu_reg_reg_shift);
11889 %}
11890 
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 // val | (-1 ^ (val >>> shift)) ==> ornw
11894 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11895                          iRegIorL2I src1, iRegIorL2I src2,
11896                          immI src3, immI_M1 src4) %{
11897   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11898   ins_cost(1.9 * INSN_COST);
11899   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11900 
11901   ins_encode %{
11902     __ ornw(as_Register($dst$$reg),
11903               as_Register($src1$$reg),
11904               as_Register($src2$$reg),
11905               Assembler::LSR,
11906               $src3$$constant & 0x1f);
11907   %}
11908 
11909   ins_pipe(ialu_reg_reg_shift);
11910 %}
11911 
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 // val | (-1 ^ (val >>> shift)) ==> orn
11915 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11916                          iRegL src1, iRegL src2,
11917                          immI src3, immL_M1 src4) %{
11918   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11919   ins_cost(1.9 * INSN_COST);
11920   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11921 
11922   ins_encode %{
11923     __ orn(as_Register($dst$$reg),
11924               as_Register($src1$$reg),
11925               as_Register($src2$$reg),
11926               Assembler::LSR,
11927               $src3$$constant & 0x3f);
11928   %}
11929 
11930   ins_pipe(ialu_reg_reg_shift);
11931 %}
11932 
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 // val | (-1 ^ (val >> shift)) ==> ornw
11936 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11937                          iRegIorL2I src1, iRegIorL2I src2,
11938                          immI src3, immI_M1 src4) %{
11939   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11940   ins_cost(1.9 * INSN_COST);
11941   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11942 
11943   ins_encode %{
11944     __ ornw(as_Register($dst$$reg),
11945               as_Register($src1$$reg),
11946               as_Register($src2$$reg),
11947               Assembler::ASR,
11948               $src3$$constant & 0x1f);
11949   %}
11950 
11951   ins_pipe(ialu_reg_reg_shift);
11952 %}
11953 
11954 // This pattern is automatically generated from aarch64_ad.m4
11955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11956 // val | (-1 ^ (val >> shift)) ==> orn
11957 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11958                          iRegL src1, iRegL src2,
11959                          immI src3, immL_M1 src4) %{
11960   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11961   ins_cost(1.9 * INSN_COST);
11962   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11963 
11964   ins_encode %{
11965     __ orn(as_Register($dst$$reg),
11966               as_Register($src1$$reg),
11967               as_Register($src2$$reg),
11968               Assembler::ASR,
11969               $src3$$constant & 0x3f);
11970   %}
11971 
11972   ins_pipe(ialu_reg_reg_shift);
11973 %}
11974 
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 // val | (-1 ^ (val ror shift)) ==> ornw
11978 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11979                          iRegIorL2I src1, iRegIorL2I src2,
11980                          immI src3, immI_M1 src4) %{
11981   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11982   ins_cost(1.9 * INSN_COST);
11983   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11984 
11985   ins_encode %{
11986     __ ornw(as_Register($dst$$reg),
11987               as_Register($src1$$reg),
11988               as_Register($src2$$reg),
11989               Assembler::ROR,
11990               $src3$$constant & 0x1f);
11991   %}
11992 
11993   ins_pipe(ialu_reg_reg_shift);
11994 %}
11995 
11996 // This pattern is automatically generated from aarch64_ad.m4
11997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11998 // val | (-1 ^ (val ror shift)) ==> orn
11999 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12000                          iRegL src1, iRegL src2,
12001                          immI src3, immL_M1 src4) %{
12002   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12003   ins_cost(1.9 * INSN_COST);
12004   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
12005 
12006   ins_encode %{
12007     __ orn(as_Register($dst$$reg),
12008               as_Register($src1$$reg),
12009               as_Register($src2$$reg),
12010               Assembler::ROR,
12011               $src3$$constant & 0x3f);
12012   %}
12013 
12014   ins_pipe(ialu_reg_reg_shift);
12015 %}
12016 
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 // val | (-1 ^ (val << shift)) ==> ornw
12020 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12021                          iRegIorL2I src1, iRegIorL2I src2,
12022                          immI src3, immI_M1 src4) %{
12023   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12024   ins_cost(1.9 * INSN_COST);
12025   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
12026 
12027   ins_encode %{
12028     __ ornw(as_Register($dst$$reg),
12029               as_Register($src1$$reg),
12030               as_Register($src2$$reg),
12031               Assembler::LSL,
12032               $src3$$constant & 0x1f);
12033   %}
12034 
12035   ins_pipe(ialu_reg_reg_shift);
12036 %}
12037 
12038 // This pattern is automatically generated from aarch64_ad.m4
12039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12040 // val | (-1 ^ (val << shift)) ==> orn
12041 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12042                          iRegL src1, iRegL src2,
12043                          immI src3, immL_M1 src4) %{
12044   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12045   ins_cost(1.9 * INSN_COST);
12046   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
12047 
12048   ins_encode %{
12049     __ orn(as_Register($dst$$reg),
12050               as_Register($src1$$reg),
12051               as_Register($src2$$reg),
12052               Assembler::LSL,
12053               $src3$$constant & 0x3f);
12054   %}
12055 
12056   ins_pipe(ialu_reg_reg_shift);
12057 %}
12058 
12059 // This pattern is automatically generated from aarch64_ad.m4
12060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12061 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12062                          iRegIorL2I src1, iRegIorL2I src2,
12063                          immI src3) %{
12064   match(Set dst (AndI src1 (URShiftI src2 src3)));
12065 
12066   ins_cost(1.9 * INSN_COST);
12067   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
12068 
12069   ins_encode %{
12070     __ andw(as_Register($dst$$reg),
12071               as_Register($src1$$reg),
12072               as_Register($src2$$reg),
12073               Assembler::LSR,
12074               $src3$$constant & 0x1f);
12075   %}
12076 
12077   ins_pipe(ialu_reg_reg_shift);
12078 %}
12079 
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12083                          iRegL src1, iRegL src2,
12084                          immI src3) %{
12085   match(Set dst (AndL src1 (URShiftL src2 src3)));
12086 
12087   ins_cost(1.9 * INSN_COST);
12088   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12089 
12090   ins_encode %{
12091     __ andr(as_Register($dst$$reg),
12092               as_Register($src1$$reg),
12093               as_Register($src2$$reg),
12094               Assembler::LSR,
12095               $src3$$constant & 0x3f);
12096   %}
12097 
12098   ins_pipe(ialu_reg_reg_shift);
12099 %}
12100 
12101 // This pattern is automatically generated from aarch64_ad.m4
12102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12103 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12104                          iRegIorL2I src1, iRegIorL2I src2,
12105                          immI src3) %{
12106   match(Set dst (AndI src1 (RShiftI src2 src3)));
12107 
12108   ins_cost(1.9 * INSN_COST);
12109   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12110 
12111   ins_encode %{
12112     __ andw(as_Register($dst$$reg),
12113               as_Register($src1$$reg),
12114               as_Register($src2$$reg),
12115               Assembler::ASR,
12116               $src3$$constant & 0x1f);
12117   %}
12118 
12119   ins_pipe(ialu_reg_reg_shift);
12120 %}
12121 
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12125                          iRegL src1, iRegL src2,
12126                          immI src3) %{
12127   match(Set dst (AndL src1 (RShiftL src2 src3)));
12128 
12129   ins_cost(1.9 * INSN_COST);
12130   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12131 
12132   ins_encode %{
12133     __ andr(as_Register($dst$$reg),
12134               as_Register($src1$$reg),
12135               as_Register($src2$$reg),
12136               Assembler::ASR,
12137               $src3$$constant & 0x3f);
12138   %}
12139 
12140   ins_pipe(ialu_reg_reg_shift);
12141 %}
12142 
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12146                          iRegIorL2I src1, iRegIorL2I src2,
12147                          immI src3) %{
12148   match(Set dst (AndI src1 (LShiftI src2 src3)));
12149 
12150   ins_cost(1.9 * INSN_COST);
12151   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12152 
12153   ins_encode %{
12154     __ andw(as_Register($dst$$reg),
12155               as_Register($src1$$reg),
12156               as_Register($src2$$reg),
12157               Assembler::LSL,
12158               $src3$$constant & 0x1f);
12159   %}
12160 
12161   ins_pipe(ialu_reg_reg_shift);
12162 %}
12163 
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12167                          iRegL src1, iRegL src2,
12168                          immI src3) %{
12169   match(Set dst (AndL src1 (LShiftL src2 src3)));
12170 
12171   ins_cost(1.9 * INSN_COST);
12172   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12173 
12174   ins_encode %{
12175     __ andr(as_Register($dst$$reg),
12176               as_Register($src1$$reg),
12177               as_Register($src2$$reg),
12178               Assembler::LSL,
12179               $src3$$constant & 0x3f);
12180   %}
12181 
12182   ins_pipe(ialu_reg_reg_shift);
12183 %}
12184 
12185 // This pattern is automatically generated from aarch64_ad.m4
12186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12187 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12188                          iRegIorL2I src1, iRegIorL2I src2,
12189                          immI src3) %{
12190   match(Set dst (AndI src1 (RotateRight src2 src3)));
12191 
12192   ins_cost(1.9 * INSN_COST);
12193   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12194 
12195   ins_encode %{
12196     __ andw(as_Register($dst$$reg),
12197               as_Register($src1$$reg),
12198               as_Register($src2$$reg),
12199               Assembler::ROR,
12200               $src3$$constant & 0x1f);
12201   %}
12202 
12203   ins_pipe(ialu_reg_reg_shift);
12204 %}
12205 
12206 // This pattern is automatically generated from aarch64_ad.m4
12207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12208 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12209                          iRegL src1, iRegL src2,
12210                          immI src3) %{
12211   match(Set dst (AndL src1 (RotateRight src2 src3)));
12212 
12213   ins_cost(1.9 * INSN_COST);
12214   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12215 
12216   ins_encode %{
12217     __ andr(as_Register($dst$$reg),
12218               as_Register($src1$$reg),
12219               as_Register($src2$$reg),
12220               Assembler::ROR,
12221               $src3$$constant & 0x3f);
12222   %}
12223 
12224   ins_pipe(ialu_reg_reg_shift);
12225 %}
12226 
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12230                          iRegIorL2I src1, iRegIorL2I src2,
12231                          immI src3) %{
12232   match(Set dst (XorI src1 (URShiftI src2 src3)));
12233 
12234   ins_cost(1.9 * INSN_COST);
12235   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12236 
12237   ins_encode %{
12238     __ eorw(as_Register($dst$$reg),
12239               as_Register($src1$$reg),
12240               as_Register($src2$$reg),
12241               Assembler::LSR,
12242               $src3$$constant & 0x1f);
12243   %}
12244 
12245   ins_pipe(ialu_reg_reg_shift);
12246 %}
12247 
12248 // This pattern is automatically generated from aarch64_ad.m4
12249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12250 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12251                          iRegL src1, iRegL src2,
12252                          immI src3) %{
12253   match(Set dst (XorL src1 (URShiftL src2 src3)));
12254 
12255   ins_cost(1.9 * INSN_COST);
12256   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12257 
12258   ins_encode %{
12259     __ eor(as_Register($dst$$reg),
12260               as_Register($src1$$reg),
12261               as_Register($src2$$reg),
12262               Assembler::LSR,
12263               $src3$$constant & 0x3f);
12264   %}
12265 
12266   ins_pipe(ialu_reg_reg_shift);
12267 %}
12268 
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12272                          iRegIorL2I src1, iRegIorL2I src2,
12273                          immI src3) %{
12274   match(Set dst (XorI src1 (RShiftI src2 src3)));
12275 
12276   ins_cost(1.9 * INSN_COST);
12277   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12278 
12279   ins_encode %{
12280     __ eorw(as_Register($dst$$reg),
12281               as_Register($src1$$reg),
12282               as_Register($src2$$reg),
12283               Assembler::ASR,
12284               $src3$$constant & 0x1f);
12285   %}
12286 
12287   ins_pipe(ialu_reg_reg_shift);
12288 %}
12289 
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12293                          iRegL src1, iRegL src2,
12294                          immI src3) %{
12295   match(Set dst (XorL src1 (RShiftL src2 src3)));
12296 
12297   ins_cost(1.9 * INSN_COST);
12298   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12299 
12300   ins_encode %{
12301     __ eor(as_Register($dst$$reg),
12302               as_Register($src1$$reg),
12303               as_Register($src2$$reg),
12304               Assembler::ASR,
12305               $src3$$constant & 0x3f);
12306   %}
12307 
12308   ins_pipe(ialu_reg_reg_shift);
12309 %}
12310 
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12314                          iRegIorL2I src1, iRegIorL2I src2,
12315                          immI src3) %{
12316   match(Set dst (XorI src1 (LShiftI src2 src3)));
12317 
12318   ins_cost(1.9 * INSN_COST);
12319   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12320 
12321   ins_encode %{
12322     __ eorw(as_Register($dst$$reg),
12323               as_Register($src1$$reg),
12324               as_Register($src2$$reg),
12325               Assembler::LSL,
12326               $src3$$constant & 0x1f);
12327   %}
12328 
12329   ins_pipe(ialu_reg_reg_shift);
12330 %}
12331 
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12335                          iRegL src1, iRegL src2,
12336                          immI src3) %{
12337   match(Set dst (XorL src1 (LShiftL src2 src3)));
12338 
12339   ins_cost(1.9 * INSN_COST);
12340   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12341 
12342   ins_encode %{
12343     __ eor(as_Register($dst$$reg),
12344               as_Register($src1$$reg),
12345               as_Register($src2$$reg),
12346               Assembler::LSL,
12347               $src3$$constant & 0x3f);
12348   %}
12349 
12350   ins_pipe(ialu_reg_reg_shift);
12351 %}
12352 
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12356                          iRegIorL2I src1, iRegIorL2I src2,
12357                          immI src3) %{
12358   match(Set dst (XorI src1 (RotateRight src2 src3)));
12359 
12360   ins_cost(1.9 * INSN_COST);
12361   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12362 
12363   ins_encode %{
12364     __ eorw(as_Register($dst$$reg),
12365               as_Register($src1$$reg),
12366               as_Register($src2$$reg),
12367               Assembler::ROR,
12368               $src3$$constant & 0x1f);
12369   %}
12370 
12371   ins_pipe(ialu_reg_reg_shift);
12372 %}
12373 
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12377                          iRegL src1, iRegL src2,
12378                          immI src3) %{
12379   match(Set dst (XorL src1 (RotateRight src2 src3)));
12380 
12381   ins_cost(1.9 * INSN_COST);
12382   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12383 
12384   ins_encode %{
12385     __ eor(as_Register($dst$$reg),
12386               as_Register($src1$$reg),
12387               as_Register($src2$$reg),
12388               Assembler::ROR,
12389               $src3$$constant & 0x3f);
12390   %}
12391 
12392   ins_pipe(ialu_reg_reg_shift);
12393 %}
12394 
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12398                          iRegIorL2I src1, iRegIorL2I src2,
12399                          immI src3) %{
12400   match(Set dst (OrI src1 (URShiftI src2 src3)));
12401 
12402   ins_cost(1.9 * INSN_COST);
12403   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12404 
12405   ins_encode %{
12406     __ orrw(as_Register($dst$$reg),
12407               as_Register($src1$$reg),
12408               as_Register($src2$$reg),
12409               Assembler::LSR,
12410               $src3$$constant & 0x1f);
12411   %}
12412 
12413   ins_pipe(ialu_reg_reg_shift);
12414 %}
12415 
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12419                          iRegL src1, iRegL src2,
12420                          immI src3) %{
12421   match(Set dst (OrL src1 (URShiftL src2 src3)));
12422 
12423   ins_cost(1.9 * INSN_COST);
12424   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12425 
12426   ins_encode %{
12427     __ orr(as_Register($dst$$reg),
12428               as_Register($src1$$reg),
12429               as_Register($src2$$reg),
12430               Assembler::LSR,
12431               $src3$$constant & 0x3f);
12432   %}
12433 
12434   ins_pipe(ialu_reg_reg_shift);
12435 %}
12436 
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12440                          iRegIorL2I src1, iRegIorL2I src2,
12441                          immI src3) %{
12442   match(Set dst (OrI src1 (RShiftI src2 src3)));
12443 
12444   ins_cost(1.9 * INSN_COST);
12445   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12446 
12447   ins_encode %{
12448     __ orrw(as_Register($dst$$reg),
12449               as_Register($src1$$reg),
12450               as_Register($src2$$reg),
12451               Assembler::ASR,
12452               $src3$$constant & 0x1f);
12453   %}
12454 
12455   ins_pipe(ialu_reg_reg_shift);
12456 %}
12457 
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12461                          iRegL src1, iRegL src2,
12462                          immI src3) %{
12463   match(Set dst (OrL src1 (RShiftL src2 src3)));
12464 
12465   ins_cost(1.9 * INSN_COST);
12466   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12467 
12468   ins_encode %{
12469     __ orr(as_Register($dst$$reg),
12470               as_Register($src1$$reg),
12471               as_Register($src2$$reg),
12472               Assembler::ASR,
12473               $src3$$constant & 0x3f);
12474   %}
12475 
12476   ins_pipe(ialu_reg_reg_shift);
12477 %}
12478 
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12482                          iRegIorL2I src1, iRegIorL2I src2,
12483                          immI src3) %{
12484   match(Set dst (OrI src1 (LShiftI src2 src3)));
12485 
12486   ins_cost(1.9 * INSN_COST);
12487   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12488 
12489   ins_encode %{
12490     __ orrw(as_Register($dst$$reg),
12491               as_Register($src1$$reg),
12492               as_Register($src2$$reg),
12493               Assembler::LSL,
12494               $src3$$constant & 0x1f);
12495   %}
12496 
12497   ins_pipe(ialu_reg_reg_shift);
12498 %}
12499 
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12503                          iRegL src1, iRegL src2,
12504                          immI src3) %{
12505   match(Set dst (OrL src1 (LShiftL src2 src3)));
12506 
12507   ins_cost(1.9 * INSN_COST);
12508   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12509 
12510   ins_encode %{
12511     __ orr(as_Register($dst$$reg),
12512               as_Register($src1$$reg),
12513               as_Register($src2$$reg),
12514               Assembler::LSL,
12515               $src3$$constant & 0x3f);
12516   %}
12517 
12518   ins_pipe(ialu_reg_reg_shift);
12519 %}
12520 
12521 // This pattern is automatically generated from aarch64_ad.m4
12522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12523 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12524                          iRegIorL2I src1, iRegIorL2I src2,
12525                          immI src3) %{
12526   match(Set dst (OrI src1 (RotateRight src2 src3)));
12527 
12528   ins_cost(1.9 * INSN_COST);
12529   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12530 
12531   ins_encode %{
12532     __ orrw(as_Register($dst$$reg),
12533               as_Register($src1$$reg),
12534               as_Register($src2$$reg),
12535               Assembler::ROR,
12536               $src3$$constant & 0x1f);
12537   %}
12538 
12539   ins_pipe(ialu_reg_reg_shift);
12540 %}
12541 
12542 // This pattern is automatically generated from aarch64_ad.m4
12543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12544 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12545                          iRegL src1, iRegL src2,
12546                          immI src3) %{
12547   match(Set dst (OrL src1 (RotateRight src2 src3)));
12548 
12549   ins_cost(1.9 * INSN_COST);
12550   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12551 
12552   ins_encode %{
12553     __ orr(as_Register($dst$$reg),
12554               as_Register($src1$$reg),
12555               as_Register($src2$$reg),
12556               Assembler::ROR,
12557               $src3$$constant & 0x3f);
12558   %}
12559 
12560   ins_pipe(ialu_reg_reg_shift);
12561 %}
12562 
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12566                          iRegIorL2I src1, iRegIorL2I src2,
12567                          immI src3) %{
12568   match(Set dst (AddI src1 (URShiftI src2 src3)));
12569 
12570   ins_cost(1.9 * INSN_COST);
12571   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12572 
12573   ins_encode %{
12574     __ addw(as_Register($dst$$reg),
12575               as_Register($src1$$reg),
12576               as_Register($src2$$reg),
12577               Assembler::LSR,
12578               $src3$$constant & 0x1f);
12579   %}
12580 
12581   ins_pipe(ialu_reg_reg_shift);
12582 %}
12583 
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12587                          iRegL src1, iRegL src2,
12588                          immI src3) %{
12589   match(Set dst (AddL src1 (URShiftL src2 src3)));
12590 
12591   ins_cost(1.9 * INSN_COST);
12592   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12593 
12594   ins_encode %{
12595     __ add(as_Register($dst$$reg),
12596               as_Register($src1$$reg),
12597               as_Register($src2$$reg),
12598               Assembler::LSR,
12599               $src3$$constant & 0x3f);
12600   %}
12601 
12602   ins_pipe(ialu_reg_reg_shift);
12603 %}
12604 
12605 // This pattern is automatically generated from aarch64_ad.m4
12606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12607 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12608                          iRegIorL2I src1, iRegIorL2I src2,
12609                          immI src3) %{
12610   match(Set dst (AddI src1 (RShiftI src2 src3)));
12611 
12612   ins_cost(1.9 * INSN_COST);
12613   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12614 
12615   ins_encode %{
12616     __ addw(as_Register($dst$$reg),
12617               as_Register($src1$$reg),
12618               as_Register($src2$$reg),
12619               Assembler::ASR,
12620               $src3$$constant & 0x1f);
12621   %}
12622 
12623   ins_pipe(ialu_reg_reg_shift);
12624 %}
12625 
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12629                          iRegL src1, iRegL src2,
12630                          immI src3) %{
12631   match(Set dst (AddL src1 (RShiftL src2 src3)));
12632 
12633   ins_cost(1.9 * INSN_COST);
12634   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12635 
12636   ins_encode %{
12637     __ add(as_Register($dst$$reg),
12638               as_Register($src1$$reg),
12639               as_Register($src2$$reg),
12640               Assembler::ASR,
12641               $src3$$constant & 0x3f);
12642   %}
12643 
12644   ins_pipe(ialu_reg_reg_shift);
12645 %}
12646 
12647 // This pattern is automatically generated from aarch64_ad.m4
12648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12649 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12650                          iRegIorL2I src1, iRegIorL2I src2,
12651                          immI src3) %{
12652   match(Set dst (AddI src1 (LShiftI src2 src3)));
12653 
12654   ins_cost(1.9 * INSN_COST);
12655   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12656 
12657   ins_encode %{
12658     __ addw(as_Register($dst$$reg),
12659               as_Register($src1$$reg),
12660               as_Register($src2$$reg),
12661               Assembler::LSL,
12662               $src3$$constant & 0x1f);
12663   %}
12664 
12665   ins_pipe(ialu_reg_reg_shift);
12666 %}
12667 
12668 // This pattern is automatically generated from aarch64_ad.m4
12669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12670 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12671                          iRegL src1, iRegL src2,
12672                          immI src3) %{
12673   match(Set dst (AddL src1 (LShiftL src2 src3)));
12674 
12675   ins_cost(1.9 * INSN_COST);
12676   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12677 
12678   ins_encode %{
12679     __ add(as_Register($dst$$reg),
12680               as_Register($src1$$reg),
12681               as_Register($src2$$reg),
12682               Assembler::LSL,
12683               $src3$$constant & 0x3f);
12684   %}
12685 
12686   ins_pipe(ialu_reg_reg_shift);
12687 %}
12688 
12689 // This pattern is automatically generated from aarch64_ad.m4
12690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12691 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12692                          iRegIorL2I src1, iRegIorL2I src2,
12693                          immI src3) %{
12694   match(Set dst (SubI src1 (URShiftI src2 src3)));
12695 
12696   ins_cost(1.9 * INSN_COST);
12697   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12698 
12699   ins_encode %{
12700     __ subw(as_Register($dst$$reg),
12701               as_Register($src1$$reg),
12702               as_Register($src2$$reg),
12703               Assembler::LSR,
12704               $src3$$constant & 0x1f);
12705   %}
12706 
12707   ins_pipe(ialu_reg_reg_shift);
12708 %}
12709 
12710 // This pattern is automatically generated from aarch64_ad.m4
12711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12712 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12713                          iRegL src1, iRegL src2,
12714                          immI src3) %{
12715   match(Set dst (SubL src1 (URShiftL src2 src3)));
12716 
12717   ins_cost(1.9 * INSN_COST);
12718   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12719 
12720   ins_encode %{
12721     __ sub(as_Register($dst$$reg),
12722               as_Register($src1$$reg),
12723               as_Register($src2$$reg),
12724               Assembler::LSR,
12725               $src3$$constant & 0x3f);
12726   %}
12727 
12728   ins_pipe(ialu_reg_reg_shift);
12729 %}
12730 
12731 // This pattern is automatically generated from aarch64_ad.m4
12732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12733 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12734                          iRegIorL2I src1, iRegIorL2I src2,
12735                          immI src3) %{
12736   match(Set dst (SubI src1 (RShiftI src2 src3)));
12737 
12738   ins_cost(1.9 * INSN_COST);
12739   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12740 
12741   ins_encode %{
12742     __ subw(as_Register($dst$$reg),
12743               as_Register($src1$$reg),
12744               as_Register($src2$$reg),
12745               Assembler::ASR,
12746               $src3$$constant & 0x1f);
12747   %}
12748 
12749   ins_pipe(ialu_reg_reg_shift);
12750 %}
12751 
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12755                          iRegL src1, iRegL src2,
12756                          immI src3) %{
12757   match(Set dst (SubL src1 (RShiftL src2 src3)));
12758 
12759   ins_cost(1.9 * INSN_COST);
12760   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12761 
12762   ins_encode %{
12763     __ sub(as_Register($dst$$reg),
12764               as_Register($src1$$reg),
12765               as_Register($src2$$reg),
12766               Assembler::ASR,
12767               $src3$$constant & 0x3f);
12768   %}
12769 
12770   ins_pipe(ialu_reg_reg_shift);
12771 %}
12772 
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12776                          iRegIorL2I src1, iRegIorL2I src2,
12777                          immI src3) %{
12778   match(Set dst (SubI src1 (LShiftI src2 src3)));
12779 
12780   ins_cost(1.9 * INSN_COST);
12781   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12782 
12783   ins_encode %{
12784     __ subw(as_Register($dst$$reg),
12785               as_Register($src1$$reg),
12786               as_Register($src2$$reg),
12787               Assembler::LSL,
12788               $src3$$constant & 0x1f);
12789   %}
12790 
12791   ins_pipe(ialu_reg_reg_shift);
12792 %}
12793 
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12797                          iRegL src1, iRegL src2,
12798                          immI src3) %{
12799   match(Set dst (SubL src1 (LShiftL src2 src3)));
12800 
12801   ins_cost(1.9 * INSN_COST);
12802   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12803 
12804   ins_encode %{
12805     __ sub(as_Register($dst$$reg),
12806               as_Register($src1$$reg),
12807               as_Register($src2$$reg),
12808               Assembler::LSL,
12809               $src3$$constant & 0x3f);
12810   %}
12811 
12812   ins_pipe(ialu_reg_reg_shift);
12813 %}
12814 
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 
12818 // Shift Left followed by Shift Right.
12819 // This idiom is used by the compiler for the i2b bytecode etc.
12820 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12821 %{
12822   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12823   ins_cost(INSN_COST * 2);
12824   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12825   ins_encode %{
12826     int lshift = $lshift_count$$constant & 63;
12827     int rshift = $rshift_count$$constant & 63;
12828     int s = 63 - lshift;
12829     int r = (rshift - lshift) & 63;
12830     __ sbfm(as_Register($dst$$reg),
12831             as_Register($src$$reg),
12832             r, s);
12833   %}
12834 
12835   ins_pipe(ialu_reg_shift);
12836 %}
12837 
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840 
12841 // Shift Left followed by Shift Right.
12842 // This idiom is used by the compiler for the i2b bytecode etc.
12843 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12844 %{
12845   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12846   ins_cost(INSN_COST * 2);
12847   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12848   ins_encode %{
12849     int lshift = $lshift_count$$constant & 31;
12850     int rshift = $rshift_count$$constant & 31;
12851     int s = 31 - lshift;
12852     int r = (rshift - lshift) & 31;
12853     __ sbfmw(as_Register($dst$$reg),
12854             as_Register($src$$reg),
12855             r, s);
12856   %}
12857 
12858   ins_pipe(ialu_reg_shift);
12859 %}
12860 
12861 // This pattern is automatically generated from aarch64_ad.m4
12862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12863 
12864 // Shift Left followed by Shift Right.
12865 // This idiom is used by the compiler for the i2b bytecode etc.
12866 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12867 %{
12868   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12869   ins_cost(INSN_COST * 2);
12870   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12871   ins_encode %{
12872     int lshift = $lshift_count$$constant & 63;
12873     int rshift = $rshift_count$$constant & 63;
12874     int s = 63 - lshift;
12875     int r = (rshift - lshift) & 63;
12876     __ ubfm(as_Register($dst$$reg),
12877             as_Register($src$$reg),
12878             r, s);
12879   %}
12880 
12881   ins_pipe(ialu_reg_shift);
12882 %}
12883 
12884 // This pattern is automatically generated from aarch64_ad.m4
12885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12886 
12887 // Shift Left followed by Shift Right.
12888 // This idiom is used by the compiler for the i2b bytecode etc.
12889 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12890 %{
12891   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12892   ins_cost(INSN_COST * 2);
12893   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12894   ins_encode %{
12895     int lshift = $lshift_count$$constant & 31;
12896     int rshift = $rshift_count$$constant & 31;
12897     int s = 31 - lshift;
12898     int r = (rshift - lshift) & 31;
12899     __ ubfmw(as_Register($dst$$reg),
12900             as_Register($src$$reg),
12901             r, s);
12902   %}
12903 
12904   ins_pipe(ialu_reg_shift);
12905 %}
12906 
12907 // Bitfield extract with shift & mask
12908 
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12912 %{
12913   match(Set dst (AndI (URShiftI src rshift) mask));
12914   // Make sure we are not going to exceed what ubfxw can do.
12915   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12916 
12917   ins_cost(INSN_COST);
12918   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12919   ins_encode %{
12920     int rshift = $rshift$$constant & 31;
12921     intptr_t mask = $mask$$constant;
12922     int width = exact_log2(mask+1);
12923     __ ubfxw(as_Register($dst$$reg),
12924             as_Register($src$$reg), rshift, width);
12925   %}
12926   ins_pipe(ialu_reg_shift);
12927 %}
12928 
12929 // This pattern is automatically generated from aarch64_ad.m4
12930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12931 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12932 %{
12933   match(Set dst (AndL (URShiftL src rshift) mask));
12934   // Make sure we are not going to exceed what ubfx can do.
12935   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12936 
12937   ins_cost(INSN_COST);
12938   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12939   ins_encode %{
12940     int rshift = $rshift$$constant & 63;
12941     intptr_t mask = $mask$$constant;
12942     int width = exact_log2_long(mask+1);
12943     __ ubfx(as_Register($dst$$reg),
12944             as_Register($src$$reg), rshift, width);
12945   %}
12946   ins_pipe(ialu_reg_shift);
12947 %}
12948 
12949 
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952 
12953 // We can use ubfx when extending an And with a mask when we know mask
12954 // is positive.  We know that because immI_bitmask guarantees it.
12955 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12956 %{
12957   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12958   // Make sure we are not going to exceed what ubfxw can do.
12959   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12960 
12961   ins_cost(INSN_COST * 2);
12962   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12963   ins_encode %{
12964     int rshift = $rshift$$constant & 31;
12965     intptr_t mask = $mask$$constant;
12966     int width = exact_log2(mask+1);
12967     __ ubfx(as_Register($dst$$reg),
12968             as_Register($src$$reg), rshift, width);
12969   %}
12970   ins_pipe(ialu_reg_shift);
12971 %}
12972 
12973 
12974 // This pattern is automatically generated from aarch64_ad.m4
12975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12976 
12977 // We can use ubfiz when masking by a positive number and then left shifting the result.
12978 // We know that the mask is positive because immI_bitmask guarantees it.
12979 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12980 %{
12981   match(Set dst (LShiftI (AndI src mask) lshift));
12982   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12983 
12984   ins_cost(INSN_COST);
12985   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12986   ins_encode %{
12987     int lshift = $lshift$$constant & 31;
12988     intptr_t mask = $mask$$constant;
12989     int width = exact_log2(mask+1);
12990     __ ubfizw(as_Register($dst$$reg),
12991           as_Register($src$$reg), lshift, width);
12992   %}
12993   ins_pipe(ialu_reg_shift);
12994 %}
12995 
12996 // This pattern is automatically generated from aarch64_ad.m4
12997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12998 
12999 // We can use ubfiz when masking by a positive number and then left shifting the result.
13000 // We know that the mask is positive because immL_bitmask guarantees it.
13001 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13002 %{
13003   match(Set dst (LShiftL (AndL src mask) lshift));
13004   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13005 
13006   ins_cost(INSN_COST);
13007   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13008   ins_encode %{
13009     int lshift = $lshift$$constant & 63;
13010     intptr_t mask = $mask$$constant;
13011     int width = exact_log2_long(mask+1);
13012     __ ubfiz(as_Register($dst$$reg),
13013           as_Register($src$$reg), lshift, width);
13014   %}
13015   ins_pipe(ialu_reg_shift);
13016 %}
13017 
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 
13021 // We can use ubfiz when masking by a positive number and then left shifting the result.
13022 // We know that the mask is positive because immI_bitmask guarantees it.
13023 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13024 %{
13025   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13026   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13027 
13028   ins_cost(INSN_COST);
13029   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13030   ins_encode %{
13031     int lshift = $lshift$$constant & 31;
13032     intptr_t mask = $mask$$constant;
13033     int width = exact_log2(mask+1);
13034     __ ubfizw(as_Register($dst$$reg),
13035           as_Register($src$$reg), lshift, width);
13036   %}
13037   ins_pipe(ialu_reg_shift);
13038 %}
13039 
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 
13043 // We can use ubfiz when masking by a positive number and then left shifting the result.
13044 // We know that the mask is positive because immL_bitmask guarantees it.
13045 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13046 %{
13047   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13048   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13049 
13050   ins_cost(INSN_COST);
13051   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13052   ins_encode %{
13053     int lshift = $lshift$$constant & 63;
13054     intptr_t mask = $mask$$constant;
13055     int width = exact_log2_long(mask+1);
13056     __ ubfiz(as_Register($dst$$reg),
13057           as_Register($src$$reg), lshift, width);
13058   %}
13059   ins_pipe(ialu_reg_shift);
13060 %}
13061 
13062 
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 
13066 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13067 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13068 %{
13069   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13070   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13071 
13072   ins_cost(INSN_COST);
13073   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13074   ins_encode %{
13075     int lshift = $lshift$$constant & 63;
13076     intptr_t mask = $mask$$constant;
13077     int width = exact_log2(mask+1);
13078     __ ubfiz(as_Register($dst$$reg),
13079              as_Register($src$$reg), lshift, width);
13080   %}
13081   ins_pipe(ialu_reg_shift);
13082 %}
13083 
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 
13087 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13088 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13089 %{
13090   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13091   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13092 
13093   ins_cost(INSN_COST);
13094   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13095   ins_encode %{
13096     int lshift = $lshift$$constant & 31;
13097     intptr_t mask = $mask$$constant;
13098     int width = exact_log2(mask+1);
13099     __ ubfiz(as_Register($dst$$reg),
13100              as_Register($src$$reg), lshift, width);
13101   %}
13102   ins_pipe(ialu_reg_shift);
13103 %}
13104 
13105 // This pattern is automatically generated from aarch64_ad.m4
13106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13107 
13108 // Can skip int2long conversions after AND with small bitmask
13109 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13110 %{
13111   match(Set dst (ConvI2L (AndI src msk)));
13112   ins_cost(INSN_COST);
13113   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13114   ins_encode %{
13115     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13116   %}
13117   ins_pipe(ialu_reg_shift);
13118 %}
13119 
13120 
13121 // Rotations
13122 
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13126 %{
13127   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13128   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13129 
13130   ins_cost(INSN_COST);
13131   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13132 
13133   ins_encode %{
13134     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13135             $rshift$$constant & 63);
13136   %}
13137   ins_pipe(ialu_reg_reg_extr);
13138 %}
13139 
13140 
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13144 %{
13145   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13146   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13147 
13148   ins_cost(INSN_COST);
13149   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13150 
13151   ins_encode %{
13152     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13153             $rshift$$constant & 31);
13154   %}
13155   ins_pipe(ialu_reg_reg_extr);
13156 %}
13157 
13158 
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13162 %{
13163   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13164   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13165 
13166   ins_cost(INSN_COST);
13167   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13168 
13169   ins_encode %{
13170     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13171             $rshift$$constant & 63);
13172   %}
13173   ins_pipe(ialu_reg_reg_extr);
13174 %}
13175 
13176 
13177 // This pattern is automatically generated from aarch64_ad.m4
13178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13179 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13180 %{
13181   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13182   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13183 
13184   ins_cost(INSN_COST);
13185   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13186 
13187   ins_encode %{
13188     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13189             $rshift$$constant & 31);
13190   %}
13191   ins_pipe(ialu_reg_reg_extr);
13192 %}
13193 
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13197 %{
13198   match(Set dst (RotateRight src shift));
13199 
13200   ins_cost(INSN_COST);
13201   format %{ "ror    $dst, $src, $shift" %}
13202 
13203   ins_encode %{
13204      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13205                $shift$$constant & 0x1f);
13206   %}
13207   ins_pipe(ialu_reg_reg_vshift);
13208 %}
13209 
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13213 %{
13214   match(Set dst (RotateRight src shift));
13215 
13216   ins_cost(INSN_COST);
13217   format %{ "ror    $dst, $src, $shift" %}
13218 
13219   ins_encode %{
13220      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13221                $shift$$constant & 0x3f);
13222   %}
13223   ins_pipe(ialu_reg_reg_vshift);
13224 %}
13225 
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13229 %{
13230   match(Set dst (RotateRight src shift));
13231 
13232   ins_cost(INSN_COST);
13233   format %{ "ror    $dst, $src, $shift" %}
13234 
13235   ins_encode %{
13236      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13237   %}
13238   ins_pipe(ialu_reg_reg_vshift);
13239 %}
13240 
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13244 %{
13245   match(Set dst (RotateRight src shift));
13246 
13247   ins_cost(INSN_COST);
13248   format %{ "ror    $dst, $src, $shift" %}
13249 
13250   ins_encode %{
13251      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13252   %}
13253   ins_pipe(ialu_reg_reg_vshift);
13254 %}
13255 
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13259 %{
13260   match(Set dst (RotateLeft src shift));
13261 
13262   ins_cost(INSN_COST);
13263   format %{ "rol    $dst, $src, $shift" %}
13264 
13265   ins_encode %{
13266      __ subw(rscratch1, zr, as_Register($shift$$reg));
13267      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13268   %}
13269   ins_pipe(ialu_reg_reg_vshift);
13270 %}
13271 
13272 // This pattern is automatically generated from aarch64_ad.m4
13273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13274 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13275 %{
13276   match(Set dst (RotateLeft src shift));
13277 
13278   ins_cost(INSN_COST);
13279   format %{ "rol    $dst, $src, $shift" %}
13280 
13281   ins_encode %{
13282      __ subw(rscratch1, zr, as_Register($shift$$reg));
13283      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13284   %}
13285   ins_pipe(ialu_reg_reg_vshift);
13286 %}
13287 
13288 
13289 // Add/subtract (extended)
13290 
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13294 %{
13295   match(Set dst (AddL src1 (ConvI2L src2)));
13296   ins_cost(INSN_COST);
13297   format %{ "add  $dst, $src1, $src2, sxtw" %}
13298 
13299    ins_encode %{
13300      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13301             as_Register($src2$$reg), ext::sxtw);
13302    %}
13303   ins_pipe(ialu_reg_reg);
13304 %}
13305 
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13309 %{
13310   match(Set dst (SubL src1 (ConvI2L src2)));
13311   ins_cost(INSN_COST);
13312   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13313 
13314    ins_encode %{
13315      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13316             as_Register($src2$$reg), ext::sxtw);
13317    %}
13318   ins_pipe(ialu_reg_reg);
13319 %}
13320 
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13324 %{
13325   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13326   ins_cost(INSN_COST);
13327   format %{ "add  $dst, $src1, $src2, sxth" %}
13328 
13329    ins_encode %{
13330      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13331             as_Register($src2$$reg), ext::sxth);
13332    %}
13333   ins_pipe(ialu_reg_reg);
13334 %}
13335 
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13339 %{
13340   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13341   ins_cost(INSN_COST);
13342   format %{ "add  $dst, $src1, $src2, sxtb" %}
13343 
13344    ins_encode %{
13345      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13346             as_Register($src2$$reg), ext::sxtb);
13347    %}
13348   ins_pipe(ialu_reg_reg);
13349 %}
13350 
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13354 %{
13355   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13356   ins_cost(INSN_COST);
13357   format %{ "add  $dst, $src1, $src2, uxtb" %}
13358 
13359    ins_encode %{
13360      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13361             as_Register($src2$$reg), ext::uxtb);
13362    %}
13363   ins_pipe(ialu_reg_reg);
13364 %}
13365 
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13369 %{
13370   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13371   ins_cost(INSN_COST);
13372   format %{ "add  $dst, $src1, $src2, sxth" %}
13373 
13374    ins_encode %{
13375      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13376             as_Register($src2$$reg), ext::sxth);
13377    %}
13378   ins_pipe(ialu_reg_reg);
13379 %}
13380 
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13384 %{
13385   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13386   ins_cost(INSN_COST);
13387   format %{ "add  $dst, $src1, $src2, sxtw" %}
13388 
13389    ins_encode %{
13390      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13391             as_Register($src2$$reg), ext::sxtw);
13392    %}
13393   ins_pipe(ialu_reg_reg);
13394 %}
13395 
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13399 %{
13400   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13401   ins_cost(INSN_COST);
13402   format %{ "add  $dst, $src1, $src2, sxtb" %}
13403 
13404    ins_encode %{
13405      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13406             as_Register($src2$$reg), ext::sxtb);
13407    %}
13408   ins_pipe(ialu_reg_reg);
13409 %}
13410 
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13414 %{
13415   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13416   ins_cost(INSN_COST);
13417   format %{ "add  $dst, $src1, $src2, uxtb" %}
13418 
13419    ins_encode %{
13420      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13421             as_Register($src2$$reg), ext::uxtb);
13422    %}
13423   ins_pipe(ialu_reg_reg);
13424 %}
13425 
13426 // This pattern is automatically generated from aarch64_ad.m4
13427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13428 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13429 %{
13430   match(Set dst (AddI src1 (AndI src2 mask)));
13431   ins_cost(INSN_COST);
13432   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13433 
13434    ins_encode %{
13435      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13436             as_Register($src2$$reg), ext::uxtb);
13437    %}
13438   ins_pipe(ialu_reg_reg);
13439 %}
13440 
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13444 %{
13445   match(Set dst (AddI src1 (AndI src2 mask)));
13446   ins_cost(INSN_COST);
13447   format %{ "addw  $dst, $src1, $src2, uxth" %}
13448 
13449    ins_encode %{
13450      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13451             as_Register($src2$$reg), ext::uxth);
13452    %}
13453   ins_pipe(ialu_reg_reg);
13454 %}
13455 
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13459 %{
13460   match(Set dst (AddL src1 (AndL src2 mask)));
13461   ins_cost(INSN_COST);
13462   format %{ "add  $dst, $src1, $src2, uxtb" %}
13463 
13464    ins_encode %{
13465      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13466             as_Register($src2$$reg), ext::uxtb);
13467    %}
13468   ins_pipe(ialu_reg_reg);
13469 %}
13470 
13471 // This pattern is automatically generated from aarch64_ad.m4
13472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13473 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13474 %{
13475   match(Set dst (AddL src1 (AndL src2 mask)));
13476   ins_cost(INSN_COST);
13477   format %{ "add  $dst, $src1, $src2, uxth" %}
13478 
13479    ins_encode %{
13480      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13481             as_Register($src2$$reg), ext::uxth);
13482    %}
13483   ins_pipe(ialu_reg_reg);
13484 %}
13485 
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13489 %{
13490   match(Set dst (AddL src1 (AndL src2 mask)));
13491   ins_cost(INSN_COST);
13492   format %{ "add  $dst, $src1, $src2, uxtw" %}
13493 
13494    ins_encode %{
13495      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13496             as_Register($src2$$reg), ext::uxtw);
13497    %}
13498   ins_pipe(ialu_reg_reg);
13499 %}
13500 
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13504 %{
13505   match(Set dst (SubI src1 (AndI src2 mask)));
13506   ins_cost(INSN_COST);
13507   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13508 
13509    ins_encode %{
13510      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13511             as_Register($src2$$reg), ext::uxtb);
13512    %}
13513   ins_pipe(ialu_reg_reg);
13514 %}
13515 
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13519 %{
13520   match(Set dst (SubI src1 (AndI src2 mask)));
13521   ins_cost(INSN_COST);
13522   format %{ "subw  $dst, $src1, $src2, uxth" %}
13523 
13524    ins_encode %{
13525      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13526             as_Register($src2$$reg), ext::uxth);
13527    %}
13528   ins_pipe(ialu_reg_reg);
13529 %}
13530 
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13534 %{
13535   match(Set dst (SubL src1 (AndL src2 mask)));
13536   ins_cost(INSN_COST);
13537   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13538 
13539    ins_encode %{
13540      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13541             as_Register($src2$$reg), ext::uxtb);
13542    %}
13543   ins_pipe(ialu_reg_reg);
13544 %}
13545 
13546 // This pattern is automatically generated from aarch64_ad.m4
13547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13548 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13549 %{
13550   match(Set dst (SubL src1 (AndL src2 mask)));
13551   ins_cost(INSN_COST);
13552   format %{ "sub  $dst, $src1, $src2, uxth" %}
13553 
13554    ins_encode %{
13555      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13556             as_Register($src2$$reg), ext::uxth);
13557    %}
13558   ins_pipe(ialu_reg_reg);
13559 %}
13560 
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13564 %{
13565   match(Set dst (SubL src1 (AndL src2 mask)));
13566   ins_cost(INSN_COST);
13567   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13568 
13569    ins_encode %{
13570      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13571             as_Register($src2$$reg), ext::uxtw);
13572    %}
13573   ins_pipe(ialu_reg_reg);
13574 %}
13575 
13576 
13577 // This pattern is automatically generated from aarch64_ad.m4
13578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13579 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13580 %{
13581   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13582   ins_cost(1.9 * INSN_COST);
13583   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13584 
13585    ins_encode %{
13586      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13587             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13588    %}
13589   ins_pipe(ialu_reg_reg_shift);
13590 %}
13591 
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13595 %{
13596   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13597   ins_cost(1.9 * INSN_COST);
13598   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13599 
13600    ins_encode %{
13601      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13602             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13603    %}
13604   ins_pipe(ialu_reg_reg_shift);
13605 %}
13606 
13607 // This pattern is automatically generated from aarch64_ad.m4
13608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13609 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13610 %{
13611   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13612   ins_cost(1.9 * INSN_COST);
13613   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13614 
13615    ins_encode %{
13616      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13617             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13618    %}
13619   ins_pipe(ialu_reg_reg_shift);
13620 %}
13621 
13622 // This pattern is automatically generated from aarch64_ad.m4
13623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13624 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13625 %{
13626   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13627   ins_cost(1.9 * INSN_COST);
13628   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13629 
13630    ins_encode %{
13631      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13632             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13633    %}
13634   ins_pipe(ialu_reg_reg_shift);
13635 %}
13636 
13637 // This pattern is automatically generated from aarch64_ad.m4
13638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13639 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13640 %{
13641   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13642   ins_cost(1.9 * INSN_COST);
13643   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13644 
13645    ins_encode %{
13646      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13647             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13648    %}
13649   ins_pipe(ialu_reg_reg_shift);
13650 %}
13651 
13652 // This pattern is automatically generated from aarch64_ad.m4
13653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13654 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13655 %{
13656   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13657   ins_cost(1.9 * INSN_COST);
13658   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13659 
13660    ins_encode %{
13661      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13662             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13663    %}
13664   ins_pipe(ialu_reg_reg_shift);
13665 %}
13666 
13667 // This pattern is automatically generated from aarch64_ad.m4
13668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13669 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13670 %{
13671   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13672   ins_cost(1.9 * INSN_COST);
13673   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13674 
13675    ins_encode %{
13676      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13677             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13678    %}
13679   ins_pipe(ialu_reg_reg_shift);
13680 %}
13681 
13682 // This pattern is automatically generated from aarch64_ad.m4
13683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13684 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13685 %{
13686   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13687   ins_cost(1.9 * INSN_COST);
13688   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13689 
13690    ins_encode %{
13691      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13692             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13693    %}
13694   ins_pipe(ialu_reg_reg_shift);
13695 %}
13696 
13697 // This pattern is automatically generated from aarch64_ad.m4
13698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13699 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13700 %{
13701   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13702   ins_cost(1.9 * INSN_COST);
13703   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13704 
13705    ins_encode %{
13706      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13707             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13708    %}
13709   ins_pipe(ialu_reg_reg_shift);
13710 %}
13711 
13712 // This pattern is automatically generated from aarch64_ad.m4
13713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13714 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13715 %{
13716   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13717   ins_cost(1.9 * INSN_COST);
13718   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13719 
13720    ins_encode %{
13721      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13722             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13723    %}
13724   ins_pipe(ialu_reg_reg_shift);
13725 %}
13726 
13727 // This pattern is automatically generated from aarch64_ad.m4
13728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13729 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13730 %{
13731   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13732   ins_cost(1.9 * INSN_COST);
13733   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13734 
13735    ins_encode %{
13736      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13737             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13738    %}
13739   ins_pipe(ialu_reg_reg_shift);
13740 %}
13741 
13742 // This pattern is automatically generated from aarch64_ad.m4
13743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13744 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13745 %{
13746   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13747   ins_cost(1.9 * INSN_COST);
13748   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13749 
13750    ins_encode %{
13751      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13752             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13753    %}
13754   ins_pipe(ialu_reg_reg_shift);
13755 %}
13756 
13757 // This pattern is automatically generated from aarch64_ad.m4
13758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13759 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13760 %{
13761   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13762   ins_cost(1.9 * INSN_COST);
13763   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13764 
13765    ins_encode %{
13766      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13767             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13768    %}
13769   ins_pipe(ialu_reg_reg_shift);
13770 %}
13771 
13772 // This pattern is automatically generated from aarch64_ad.m4
13773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13774 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13775 %{
13776   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13777   ins_cost(1.9 * INSN_COST);
13778   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13779 
13780    ins_encode %{
13781      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13782             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13783    %}
13784   ins_pipe(ialu_reg_reg_shift);
13785 %}
13786 
13787 // This pattern is automatically generated from aarch64_ad.m4
13788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13789 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13790 %{
13791   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13792   ins_cost(1.9 * INSN_COST);
13793   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13794 
13795    ins_encode %{
13796      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13797             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13798    %}
13799   ins_pipe(ialu_reg_reg_shift);
13800 %}
13801 
13802 // This pattern is automatically generated from aarch64_ad.m4
13803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13804 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13805 %{
13806   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13807   ins_cost(1.9 * INSN_COST);
13808   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13809 
13810    ins_encode %{
13811      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13812             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13813    %}
13814   ins_pipe(ialu_reg_reg_shift);
13815 %}
13816 
13817 // This pattern is automatically generated from aarch64_ad.m4
13818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13819 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13820 %{
13821   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13822   ins_cost(1.9 * INSN_COST);
13823   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13824 
13825    ins_encode %{
13826      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13827             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13828    %}
13829   ins_pipe(ialu_reg_reg_shift);
13830 %}
13831 
13832 // This pattern is automatically generated from aarch64_ad.m4
13833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13834 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13835 %{
13836   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13837   ins_cost(1.9 * INSN_COST);
13838   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13839 
13840    ins_encode %{
13841      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13842             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13843    %}
13844   ins_pipe(ialu_reg_reg_shift);
13845 %}
13846 
13847 // This pattern is automatically generated from aarch64_ad.m4
13848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13849 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13850 %{
13851   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13852   ins_cost(1.9 * INSN_COST);
13853   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13854 
13855    ins_encode %{
13856      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13857             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13858    %}
13859   ins_pipe(ialu_reg_reg_shift);
13860 %}
13861 
13862 // This pattern is automatically generated from aarch64_ad.m4
13863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13864 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13865 %{
13866   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13867   ins_cost(1.9 * INSN_COST);
13868   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13869 
13870    ins_encode %{
13871      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13872             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13873    %}
13874   ins_pipe(ialu_reg_reg_shift);
13875 %}
13876 
13877 // This pattern is automatically generated from aarch64_ad.m4
13878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13879 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13880 %{
13881   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13882   ins_cost(1.9 * INSN_COST);
13883   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13884 
13885    ins_encode %{
13886      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13887             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13888    %}
13889   ins_pipe(ialu_reg_reg_shift);
13890 %}
13891 
13892 // This pattern is automatically generated from aarch64_ad.m4
13893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13894 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13895 %{
13896   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13897   ins_cost(1.9 * INSN_COST);
13898   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13899 
13900    ins_encode %{
13901      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13902             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13903    %}
13904   ins_pipe(ialu_reg_reg_shift);
13905 %}
13906 
13907 // This pattern is automatically generated from aarch64_ad.m4
13908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13909 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13910 %{
13911   effect(DEF dst, USE src1, USE src2, USE cr);
13912   ins_cost(INSN_COST * 2);
13913   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13914 
13915   ins_encode %{
13916     __ cselw($dst$$Register,
13917              $src1$$Register,
13918              $src2$$Register,
13919              Assembler::LT);
13920   %}
13921   ins_pipe(icond_reg_reg);
13922 %}
13923 
13924 // This pattern is automatically generated from aarch64_ad.m4
13925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13926 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13927 %{
13928   effect(DEF dst, USE src1, USE src2, USE cr);
13929   ins_cost(INSN_COST * 2);
13930   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13931 
13932   ins_encode %{
13933     __ cselw($dst$$Register,
13934              $src1$$Register,
13935              $src2$$Register,
13936              Assembler::GT);
13937   %}
13938   ins_pipe(icond_reg_reg);
13939 %}
13940 
13941 // This pattern is automatically generated from aarch64_ad.m4
13942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13943 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13944 %{
13945   effect(DEF dst, USE src1, USE cr);
13946   ins_cost(INSN_COST * 2);
13947   format %{ "cselw $dst, $src1, zr lt\t"  %}
13948 
13949   ins_encode %{
13950     __ cselw($dst$$Register,
13951              $src1$$Register,
13952              zr,
13953              Assembler::LT);
13954   %}
13955   ins_pipe(icond_reg);
13956 %}
13957 
13958 // This pattern is automatically generated from aarch64_ad.m4
13959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13960 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13961 %{
13962   effect(DEF dst, USE src1, USE cr);
13963   ins_cost(INSN_COST * 2);
13964   format %{ "cselw $dst, $src1, zr gt\t"  %}
13965 
13966   ins_encode %{
13967     __ cselw($dst$$Register,
13968              $src1$$Register,
13969              zr,
13970              Assembler::GT);
13971   %}
13972   ins_pipe(icond_reg);
13973 %}
13974 
13975 // This pattern is automatically generated from aarch64_ad.m4
13976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13977 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13978 %{
13979   effect(DEF dst, USE src1, USE cr);
13980   ins_cost(INSN_COST * 2);
13981   format %{ "csincw $dst, $src1, zr le\t"  %}
13982 
13983   ins_encode %{
13984     __ csincw($dst$$Register,
13985              $src1$$Register,
13986              zr,
13987              Assembler::LE);
13988   %}
13989   ins_pipe(icond_reg);
13990 %}
13991 
13992 // This pattern is automatically generated from aarch64_ad.m4
13993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13994 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13995 %{
13996   effect(DEF dst, USE src1, USE cr);
13997   ins_cost(INSN_COST * 2);
13998   format %{ "csincw $dst, $src1, zr gt\t"  %}
13999 
14000   ins_encode %{
14001     __ csincw($dst$$Register,
14002              $src1$$Register,
14003              zr,
14004              Assembler::GT);
14005   %}
14006   ins_pipe(icond_reg);
14007 %}
14008 
14009 // This pattern is automatically generated from aarch64_ad.m4
14010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14011 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14012 %{
14013   effect(DEF dst, USE src1, USE cr);
14014   ins_cost(INSN_COST * 2);
14015   format %{ "csinvw $dst, $src1, zr lt\t"  %}
14016 
14017   ins_encode %{
14018     __ csinvw($dst$$Register,
14019              $src1$$Register,
14020              zr,
14021              Assembler::LT);
14022   %}
14023   ins_pipe(icond_reg);
14024 %}
14025 
14026 // This pattern is automatically generated from aarch64_ad.m4
14027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14028 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14029 %{
14030   effect(DEF dst, USE src1, USE cr);
14031   ins_cost(INSN_COST * 2);
14032   format %{ "csinvw $dst, $src1, zr ge\t"  %}
14033 
14034   ins_encode %{
14035     __ csinvw($dst$$Register,
14036              $src1$$Register,
14037              zr,
14038              Assembler::GE);
14039   %}
14040   ins_pipe(icond_reg);
14041 %}
14042 
14043 // This pattern is automatically generated from aarch64_ad.m4
14044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14045 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14046 %{
14047   match(Set dst (MinI src imm));
14048   ins_cost(INSN_COST * 3);
14049   expand %{
14050     rFlagsReg cr;
14051     compI_reg_imm0(cr, src);
14052     cmovI_reg_imm0_lt(dst, src, cr);
14053   %}
14054 %}
14055 
14056 // This pattern is automatically generated from aarch64_ad.m4
14057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14058 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14059 %{
14060   match(Set dst (MinI imm src));
14061   ins_cost(INSN_COST * 3);
14062   expand %{
14063     rFlagsReg cr;
14064     compI_reg_imm0(cr, src);
14065     cmovI_reg_imm0_lt(dst, src, cr);
14066   %}
14067 %}
14068 
14069 // This pattern is automatically generated from aarch64_ad.m4
14070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14071 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14072 %{
14073   match(Set dst (MinI src imm));
14074   ins_cost(INSN_COST * 3);
14075   expand %{
14076     rFlagsReg cr;
14077     compI_reg_imm0(cr, src);
14078     cmovI_reg_imm1_le(dst, src, cr);
14079   %}
14080 %}
14081 
14082 // This pattern is automatically generated from aarch64_ad.m4
14083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14084 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14085 %{
14086   match(Set dst (MinI imm src));
14087   ins_cost(INSN_COST * 3);
14088   expand %{
14089     rFlagsReg cr;
14090     compI_reg_imm0(cr, src);
14091     cmovI_reg_imm1_le(dst, src, cr);
14092   %}
14093 %}
14094 
14095 // This pattern is automatically generated from aarch64_ad.m4
14096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14097 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14098 %{
14099   match(Set dst (MinI src imm));
14100   ins_cost(INSN_COST * 3);
14101   expand %{
14102     rFlagsReg cr;
14103     compI_reg_imm0(cr, src);
14104     cmovI_reg_immM1_lt(dst, src, cr);
14105   %}
14106 %}
14107 
14108 // This pattern is automatically generated from aarch64_ad.m4
14109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14110 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14111 %{
14112   match(Set dst (MinI imm src));
14113   ins_cost(INSN_COST * 3);
14114   expand %{
14115     rFlagsReg cr;
14116     compI_reg_imm0(cr, src);
14117     cmovI_reg_immM1_lt(dst, src, cr);
14118   %}
14119 %}
14120 
14121 // This pattern is automatically generated from aarch64_ad.m4
14122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14123 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14124 %{
14125   match(Set dst (MaxI src imm));
14126   ins_cost(INSN_COST * 3);
14127   expand %{
14128     rFlagsReg cr;
14129     compI_reg_imm0(cr, src);
14130     cmovI_reg_imm0_gt(dst, src, cr);
14131   %}
14132 %}
14133 
14134 // This pattern is automatically generated from aarch64_ad.m4
14135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14136 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14137 %{
14138   match(Set dst (MaxI imm src));
14139   ins_cost(INSN_COST * 3);
14140   expand %{
14141     rFlagsReg cr;
14142     compI_reg_imm0(cr, src);
14143     cmovI_reg_imm0_gt(dst, src, cr);
14144   %}
14145 %}
14146 
14147 // This pattern is automatically generated from aarch64_ad.m4
14148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14149 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14150 %{
14151   match(Set dst (MaxI src imm));
14152   ins_cost(INSN_COST * 3);
14153   expand %{
14154     rFlagsReg cr;
14155     compI_reg_imm0(cr, src);
14156     cmovI_reg_imm1_gt(dst, src, cr);
14157   %}
14158 %}
14159 
14160 // This pattern is automatically generated from aarch64_ad.m4
14161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14162 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14163 %{
14164   match(Set dst (MaxI imm src));
14165   ins_cost(INSN_COST * 3);
14166   expand %{
14167     rFlagsReg cr;
14168     compI_reg_imm0(cr, src);
14169     cmovI_reg_imm1_gt(dst, src, cr);
14170   %}
14171 %}
14172 
14173 // This pattern is automatically generated from aarch64_ad.m4
14174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14175 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14176 %{
14177   match(Set dst (MaxI src imm));
14178   ins_cost(INSN_COST * 3);
14179   expand %{
14180     rFlagsReg cr;
14181     compI_reg_imm0(cr, src);
14182     cmovI_reg_immM1_ge(dst, src, cr);
14183   %}
14184 %}
14185 
14186 // This pattern is automatically generated from aarch64_ad.m4
14187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14188 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14189 %{
14190   match(Set dst (MaxI imm src));
14191   ins_cost(INSN_COST * 3);
14192   expand %{
14193     rFlagsReg cr;
14194     compI_reg_imm0(cr, src);
14195     cmovI_reg_immM1_ge(dst, src, cr);
14196   %}
14197 %}
14198 
14199 // This pattern is automatically generated from aarch64_ad.m4
14200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14201 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14202 %{
14203   match(Set dst (ReverseI src));
14204   ins_cost(INSN_COST);
14205   format %{ "rbitw  $dst, $src" %}
14206   ins_encode %{
14207     __ rbitw($dst$$Register, $src$$Register);
14208   %}
14209   ins_pipe(ialu_reg);
14210 %}
14211 
14212 // This pattern is automatically generated from aarch64_ad.m4
14213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14214 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14215 %{
14216   match(Set dst (ReverseL src));
14217   ins_cost(INSN_COST);
14218   format %{ "rbit  $dst, $src" %}
14219   ins_encode %{
14220     __ rbit($dst$$Register, $src$$Register);
14221   %}
14222   ins_pipe(ialu_reg);
14223 %}
14224 
14225 
14226 // END This section of the file is automatically generated. Do not edit --------------
14227 
14228 
14229 // ============================================================================
14230 // Floating Point Arithmetic Instructions
14231 
14232 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14233   match(Set dst (AddF src1 src2));
14234 
14235   ins_cost(INSN_COST * 5);
14236   format %{ "fadds   $dst, $src1, $src2" %}
14237 
14238   ins_encode %{
14239     __ fadds(as_FloatRegister($dst$$reg),
14240              as_FloatRegister($src1$$reg),
14241              as_FloatRegister($src2$$reg));
14242   %}
14243 
14244   ins_pipe(fp_dop_reg_reg_s);
14245 %}
14246 
14247 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14248   match(Set dst (AddD src1 src2));
14249 
14250   ins_cost(INSN_COST * 5);
14251   format %{ "faddd   $dst, $src1, $src2" %}
14252 
14253   ins_encode %{
14254     __ faddd(as_FloatRegister($dst$$reg),
14255              as_FloatRegister($src1$$reg),
14256              as_FloatRegister($src2$$reg));
14257   %}
14258 
14259   ins_pipe(fp_dop_reg_reg_d);
14260 %}
14261 
14262 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14263   match(Set dst (SubF src1 src2));
14264 
14265   ins_cost(INSN_COST * 5);
14266   format %{ "fsubs   $dst, $src1, $src2" %}
14267 
14268   ins_encode %{
14269     __ fsubs(as_FloatRegister($dst$$reg),
14270              as_FloatRegister($src1$$reg),
14271              as_FloatRegister($src2$$reg));
14272   %}
14273 
14274   ins_pipe(fp_dop_reg_reg_s);
14275 %}
14276 
14277 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14278   match(Set dst (SubD src1 src2));
14279 
14280   ins_cost(INSN_COST * 5);
14281   format %{ "fsubd   $dst, $src1, $src2" %}
14282 
14283   ins_encode %{
14284     __ fsubd(as_FloatRegister($dst$$reg),
14285              as_FloatRegister($src1$$reg),
14286              as_FloatRegister($src2$$reg));
14287   %}
14288 
14289   ins_pipe(fp_dop_reg_reg_d);
14290 %}
14291 
14292 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14293   match(Set dst (MulF src1 src2));
14294 
14295   ins_cost(INSN_COST * 6);
14296   format %{ "fmuls   $dst, $src1, $src2" %}
14297 
14298   ins_encode %{
14299     __ fmuls(as_FloatRegister($dst$$reg),
14300              as_FloatRegister($src1$$reg),
14301              as_FloatRegister($src2$$reg));
14302   %}
14303 
14304   ins_pipe(fp_dop_reg_reg_s);
14305 %}
14306 
14307 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14308   match(Set dst (MulD src1 src2));
14309 
14310   ins_cost(INSN_COST * 6);
14311   format %{ "fmuld   $dst, $src1, $src2" %}
14312 
14313   ins_encode %{
14314     __ fmuld(as_FloatRegister($dst$$reg),
14315              as_FloatRegister($src1$$reg),
14316              as_FloatRegister($src2$$reg));
14317   %}
14318 
14319   ins_pipe(fp_dop_reg_reg_d);
14320 %}
14321 
14322 // src1 * src2 + src3
14323 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14324   predicate(UseFMA);
14325   match(Set dst (FmaF src3 (Binary src1 src2)));
14326 
14327   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14328 
14329   ins_encode %{
14330     __ fmadds(as_FloatRegister($dst$$reg),
14331              as_FloatRegister($src1$$reg),
14332              as_FloatRegister($src2$$reg),
14333              as_FloatRegister($src3$$reg));
14334   %}
14335 
14336   ins_pipe(pipe_class_default);
14337 %}
14338 
14339 // src1 * src2 + src3
14340 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14341   predicate(UseFMA);
14342   match(Set dst (FmaD src3 (Binary src1 src2)));
14343 
14344   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14345 
14346   ins_encode %{
14347     __ fmaddd(as_FloatRegister($dst$$reg),
14348              as_FloatRegister($src1$$reg),
14349              as_FloatRegister($src2$$reg),
14350              as_FloatRegister($src3$$reg));
14351   %}
14352 
14353   ins_pipe(pipe_class_default);
14354 %}
14355 
14356 // -src1 * src2 + src3
14357 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14358   predicate(UseFMA);
14359   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14360   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14361 
14362   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14363 
14364   ins_encode %{
14365     __ fmsubs(as_FloatRegister($dst$$reg),
14366               as_FloatRegister($src1$$reg),
14367               as_FloatRegister($src2$$reg),
14368               as_FloatRegister($src3$$reg));
14369   %}
14370 
14371   ins_pipe(pipe_class_default);
14372 %}
14373 
14374 // -src1 * src2 + src3
14375 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14376   predicate(UseFMA);
14377   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14378   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14379 
14380   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14381 
14382   ins_encode %{
14383     __ fmsubd(as_FloatRegister($dst$$reg),
14384               as_FloatRegister($src1$$reg),
14385               as_FloatRegister($src2$$reg),
14386               as_FloatRegister($src3$$reg));
14387   %}
14388 
14389   ins_pipe(pipe_class_default);
14390 %}
14391 
14392 // -src1 * src2 - src3
14393 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14394   predicate(UseFMA);
14395   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14396   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14397 
14398   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14399 
14400   ins_encode %{
14401     __ fnmadds(as_FloatRegister($dst$$reg),
14402                as_FloatRegister($src1$$reg),
14403                as_FloatRegister($src2$$reg),
14404                as_FloatRegister($src3$$reg));
14405   %}
14406 
14407   ins_pipe(pipe_class_default);
14408 %}
14409 
14410 // -src1 * src2 - src3
14411 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14412   predicate(UseFMA);
14413   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14414   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14415 
14416   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14417 
14418   ins_encode %{
14419     __ fnmaddd(as_FloatRegister($dst$$reg),
14420                as_FloatRegister($src1$$reg),
14421                as_FloatRegister($src2$$reg),
14422                as_FloatRegister($src3$$reg));
14423   %}
14424 
14425   ins_pipe(pipe_class_default);
14426 %}
14427 
14428 // src1 * src2 - src3
14429 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14430   predicate(UseFMA);
14431   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14432 
14433   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14434 
14435   ins_encode %{
14436     __ fnmsubs(as_FloatRegister($dst$$reg),
14437                as_FloatRegister($src1$$reg),
14438                as_FloatRegister($src2$$reg),
14439                as_FloatRegister($src3$$reg));
14440   %}
14441 
14442   ins_pipe(pipe_class_default);
14443 %}
14444 
14445 // src1 * src2 - src3
14446 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14447   predicate(UseFMA);
14448   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14449 
14450   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14451 
14452   ins_encode %{
14453   // n.b. insn name should be fnmsubd
14454     __ fnmsub(as_FloatRegister($dst$$reg),
14455               as_FloatRegister($src1$$reg),
14456               as_FloatRegister($src2$$reg),
14457               as_FloatRegister($src3$$reg));
14458   %}
14459 
14460   ins_pipe(pipe_class_default);
14461 %}
14462 
14463 
14464 // Math.max(FF)F
14465 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14466   match(Set dst (MaxF src1 src2));
14467 
14468   format %{ "fmaxs   $dst, $src1, $src2" %}
14469   ins_encode %{
14470     __ fmaxs(as_FloatRegister($dst$$reg),
14471              as_FloatRegister($src1$$reg),
14472              as_FloatRegister($src2$$reg));
14473   %}
14474 
14475   ins_pipe(fp_dop_reg_reg_s);
14476 %}
14477 
14478 // Math.min(FF)F
14479 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14480   match(Set dst (MinF src1 src2));
14481 
14482   format %{ "fmins   $dst, $src1, $src2" %}
14483   ins_encode %{
14484     __ fmins(as_FloatRegister($dst$$reg),
14485              as_FloatRegister($src1$$reg),
14486              as_FloatRegister($src2$$reg));
14487   %}
14488 
14489   ins_pipe(fp_dop_reg_reg_s);
14490 %}
14491 
14492 // Math.max(DD)D
14493 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14494   match(Set dst (MaxD src1 src2));
14495 
14496   format %{ "fmaxd   $dst, $src1, $src2" %}
14497   ins_encode %{
14498     __ fmaxd(as_FloatRegister($dst$$reg),
14499              as_FloatRegister($src1$$reg),
14500              as_FloatRegister($src2$$reg));
14501   %}
14502 
14503   ins_pipe(fp_dop_reg_reg_d);
14504 %}
14505 
14506 // Math.min(DD)D
14507 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14508   match(Set dst (MinD src1 src2));
14509 
14510   format %{ "fmind   $dst, $src1, $src2" %}
14511   ins_encode %{
14512     __ fmind(as_FloatRegister($dst$$reg),
14513              as_FloatRegister($src1$$reg),
14514              as_FloatRegister($src2$$reg));
14515   %}
14516 
14517   ins_pipe(fp_dop_reg_reg_d);
14518 %}
14519 
14520 
14521 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14522   match(Set dst (DivF src1  src2));
14523 
14524   ins_cost(INSN_COST * 18);
14525   format %{ "fdivs   $dst, $src1, $src2" %}
14526 
14527   ins_encode %{
14528     __ fdivs(as_FloatRegister($dst$$reg),
14529              as_FloatRegister($src1$$reg),
14530              as_FloatRegister($src2$$reg));
14531   %}
14532 
14533   ins_pipe(fp_div_s);
14534 %}
14535 
14536 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14537   match(Set dst (DivD src1  src2));
14538 
14539   ins_cost(INSN_COST * 32);
14540   format %{ "fdivd   $dst, $src1, $src2" %}
14541 
14542   ins_encode %{
14543     __ fdivd(as_FloatRegister($dst$$reg),
14544              as_FloatRegister($src1$$reg),
14545              as_FloatRegister($src2$$reg));
14546   %}
14547 
14548   ins_pipe(fp_div_d);
14549 %}
14550 
14551 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14552   match(Set dst (NegF src));
14553 
14554   ins_cost(INSN_COST * 3);
14555   format %{ "fneg   $dst, $src" %}
14556 
14557   ins_encode %{
14558     __ fnegs(as_FloatRegister($dst$$reg),
14559              as_FloatRegister($src$$reg));
14560   %}
14561 
14562   ins_pipe(fp_uop_s);
14563 %}
14564 
14565 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14566   match(Set dst (NegD src));
14567 
14568   ins_cost(INSN_COST * 3);
14569   format %{ "fnegd   $dst, $src" %}
14570 
14571   ins_encode %{
14572     __ fnegd(as_FloatRegister($dst$$reg),
14573              as_FloatRegister($src$$reg));
14574   %}
14575 
14576   ins_pipe(fp_uop_d);
14577 %}
14578 
14579 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14580 %{
14581   match(Set dst (AbsI src));
14582 
14583   effect(KILL cr);
14584   ins_cost(INSN_COST * 2);
14585   format %{ "cmpw  $src, zr\n\t"
14586             "cnegw $dst, $src, Assembler::LT\t# int abs"
14587   %}
14588 
14589   ins_encode %{
14590     __ cmpw(as_Register($src$$reg), zr);
14591     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14592   %}
14593   ins_pipe(pipe_class_default);
14594 %}
14595 
14596 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14597 %{
14598   match(Set dst (AbsL src));
14599 
14600   effect(KILL cr);
14601   ins_cost(INSN_COST * 2);
14602   format %{ "cmp  $src, zr\n\t"
14603             "cneg $dst, $src, Assembler::LT\t# long abs"
14604   %}
14605 
14606   ins_encode %{
14607     __ cmp(as_Register($src$$reg), zr);
14608     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14609   %}
14610   ins_pipe(pipe_class_default);
14611 %}
14612 
14613 instruct absF_reg(vRegF dst, vRegF src) %{
14614   match(Set dst (AbsF src));
14615 
14616   ins_cost(INSN_COST * 3);
14617   format %{ "fabss   $dst, $src" %}
14618   ins_encode %{
14619     __ fabss(as_FloatRegister($dst$$reg),
14620              as_FloatRegister($src$$reg));
14621   %}
14622 
14623   ins_pipe(fp_uop_s);
14624 %}
14625 
14626 instruct absD_reg(vRegD dst, vRegD src) %{
14627   match(Set dst (AbsD src));
14628 
14629   ins_cost(INSN_COST * 3);
14630   format %{ "fabsd   $dst, $src" %}
14631   ins_encode %{
14632     __ fabsd(as_FloatRegister($dst$$reg),
14633              as_FloatRegister($src$$reg));
14634   %}
14635 
14636   ins_pipe(fp_uop_d);
14637 %}
14638 
14639 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14640   match(Set dst (AbsF (SubF src1 src2)));
14641 
14642   ins_cost(INSN_COST * 3);
14643   format %{ "fabds   $dst, $src1, $src2" %}
14644   ins_encode %{
14645     __ fabds(as_FloatRegister($dst$$reg),
14646              as_FloatRegister($src1$$reg),
14647              as_FloatRegister($src2$$reg));
14648   %}
14649 
14650   ins_pipe(fp_uop_s);
14651 %}
14652 
14653 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14654   match(Set dst (AbsD (SubD src1 src2)));
14655 
14656   ins_cost(INSN_COST * 3);
14657   format %{ "fabdd   $dst, $src1, $src2" %}
14658   ins_encode %{
14659     __ fabdd(as_FloatRegister($dst$$reg),
14660              as_FloatRegister($src1$$reg),
14661              as_FloatRegister($src2$$reg));
14662   %}
14663 
14664   ins_pipe(fp_uop_d);
14665 %}
14666 
14667 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14668   match(Set dst (SqrtD src));
14669 
14670   ins_cost(INSN_COST * 50);
14671   format %{ "fsqrtd  $dst, $src" %}
14672   ins_encode %{
14673     __ fsqrtd(as_FloatRegister($dst$$reg),
14674              as_FloatRegister($src$$reg));
14675   %}
14676 
14677   ins_pipe(fp_div_s);
14678 %}
14679 
14680 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14681   match(Set dst (SqrtF src));
14682 
14683   ins_cost(INSN_COST * 50);
14684   format %{ "fsqrts  $dst, $src" %}
14685   ins_encode %{
14686     __ fsqrts(as_FloatRegister($dst$$reg),
14687              as_FloatRegister($src$$reg));
14688   %}
14689 
14690   ins_pipe(fp_div_d);
14691 %}
14692 
14693 // Math.rint, floor, ceil
14694 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14695   match(Set dst (RoundDoubleMode src rmode));
14696   format %{ "frint  $dst, $src, $rmode" %}
14697   ins_encode %{
14698     switch ($rmode$$constant) {
14699       case RoundDoubleModeNode::rmode_rint:
14700         __ frintnd(as_FloatRegister($dst$$reg),
14701                    as_FloatRegister($src$$reg));
14702         break;
14703       case RoundDoubleModeNode::rmode_floor:
14704         __ frintmd(as_FloatRegister($dst$$reg),
14705                    as_FloatRegister($src$$reg));
14706         break;
14707       case RoundDoubleModeNode::rmode_ceil:
14708         __ frintpd(as_FloatRegister($dst$$reg),
14709                    as_FloatRegister($src$$reg));
14710         break;
14711     }
14712   %}
14713   ins_pipe(fp_uop_d);
14714 %}
14715 
14716 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14717   match(Set dst (CopySignD src1 (Binary src2 zero)));
14718   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14719   format %{ "CopySignD  $dst $src1 $src2" %}
14720   ins_encode %{
14721     FloatRegister dst = as_FloatRegister($dst$$reg),
14722                   src1 = as_FloatRegister($src1$$reg),
14723                   src2 = as_FloatRegister($src2$$reg),
14724                   zero = as_FloatRegister($zero$$reg);
14725     __ fnegd(dst, zero);
14726     __ bsl(dst, __ T8B, src2, src1);
14727   %}
14728   ins_pipe(fp_uop_d);
14729 %}
14730 
14731 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14732   match(Set dst (CopySignF src1 src2));
14733   effect(TEMP_DEF dst, USE src1, USE src2);
14734   format %{ "CopySignF  $dst $src1 $src2" %}
14735   ins_encode %{
14736     FloatRegister dst = as_FloatRegister($dst$$reg),
14737                   src1 = as_FloatRegister($src1$$reg),
14738                   src2 = as_FloatRegister($src2$$reg);
14739     __ movi(dst, __ T2S, 0x80, 24);
14740     __ bsl(dst, __ T8B, src2, src1);
14741   %}
14742   ins_pipe(fp_uop_d);
14743 %}
14744 
14745 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14746   match(Set dst (SignumD src (Binary zero one)));
14747   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14748   format %{ "signumD  $dst, $src" %}
14749   ins_encode %{
14750     FloatRegister src = as_FloatRegister($src$$reg),
14751                   dst = as_FloatRegister($dst$$reg),
14752                   zero = as_FloatRegister($zero$$reg),
14753                   one = as_FloatRegister($one$$reg);
14754     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14755     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14756     // Bit selection instruction gets bit from "one" for each enabled bit in
14757     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14758     // NaN the whole "src" will be copied because "dst" is zero. For all other
14759     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14760     // from "src", and all other bits are copied from 1.0.
14761     __ bsl(dst, __ T8B, one, src);
14762   %}
14763   ins_pipe(fp_uop_d);
14764 %}
14765 
14766 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14767   match(Set dst (SignumF src (Binary zero one)));
14768   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14769   format %{ "signumF  $dst, $src" %}
14770   ins_encode %{
14771     FloatRegister src = as_FloatRegister($src$$reg),
14772                   dst = as_FloatRegister($dst$$reg),
14773                   zero = as_FloatRegister($zero$$reg),
14774                   one = as_FloatRegister($one$$reg);
14775     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14776     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14777     // Bit selection instruction gets bit from "one" for each enabled bit in
14778     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14779     // NaN the whole "src" will be copied because "dst" is zero. For all other
14780     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14781     // from "src", and all other bits are copied from 1.0.
14782     __ bsl(dst, __ T8B, one, src);
14783   %}
14784   ins_pipe(fp_uop_d);
14785 %}
14786 
14787 instruct onspinwait() %{
14788   match(OnSpinWait);
14789   ins_cost(INSN_COST);
14790 
14791   format %{ "onspinwait" %}
14792 
14793   ins_encode %{
14794     __ spin_wait();
14795   %}
14796   ins_pipe(pipe_class_empty);
14797 %}
14798 
14799 // ============================================================================
14800 // Logical Instructions
14801 
14802 // Integer Logical Instructions
14803 
14804 // And Instructions
14805 
14806 
14807 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14808   match(Set dst (AndI src1 src2));
14809 
14810   format %{ "andw  $dst, $src1, $src2\t# int" %}
14811 
14812   ins_cost(INSN_COST);
14813   ins_encode %{
14814     __ andw(as_Register($dst$$reg),
14815             as_Register($src1$$reg),
14816             as_Register($src2$$reg));
14817   %}
14818 
14819   ins_pipe(ialu_reg_reg);
14820 %}
14821 
14822 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14823   match(Set dst (AndI src1 src2));
14824 
14825   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14826 
14827   ins_cost(INSN_COST);
14828   ins_encode %{
14829     __ andw(as_Register($dst$$reg),
14830             as_Register($src1$$reg),
14831             (uint64_t)($src2$$constant));
14832   %}
14833 
14834   ins_pipe(ialu_reg_imm);
14835 %}
14836 
14837 // Or Instructions
14838 
14839 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14840   match(Set dst (OrI src1 src2));
14841 
14842   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14843 
14844   ins_cost(INSN_COST);
14845   ins_encode %{
14846     __ orrw(as_Register($dst$$reg),
14847             as_Register($src1$$reg),
14848             as_Register($src2$$reg));
14849   %}
14850 
14851   ins_pipe(ialu_reg_reg);
14852 %}
14853 
14854 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14855   match(Set dst (OrI src1 src2));
14856 
14857   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14858 
14859   ins_cost(INSN_COST);
14860   ins_encode %{
14861     __ orrw(as_Register($dst$$reg),
14862             as_Register($src1$$reg),
14863             (uint64_t)($src2$$constant));
14864   %}
14865 
14866   ins_pipe(ialu_reg_imm);
14867 %}
14868 
14869 // Xor Instructions
14870 
14871 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14872   match(Set dst (XorI src1 src2));
14873 
14874   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14875 
14876   ins_cost(INSN_COST);
14877   ins_encode %{
14878     __ eorw(as_Register($dst$$reg),
14879             as_Register($src1$$reg),
14880             as_Register($src2$$reg));
14881   %}
14882 
14883   ins_pipe(ialu_reg_reg);
14884 %}
14885 
14886 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14887   match(Set dst (XorI src1 src2));
14888 
14889   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14890 
14891   ins_cost(INSN_COST);
14892   ins_encode %{
14893     __ eorw(as_Register($dst$$reg),
14894             as_Register($src1$$reg),
14895             (uint64_t)($src2$$constant));
14896   %}
14897 
14898   ins_pipe(ialu_reg_imm);
14899 %}
14900 
14901 // Long Logical Instructions
14902 // TODO
14903 
14904 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14905   match(Set dst (AndL src1 src2));
14906 
14907   format %{ "and  $dst, $src1, $src2\t# int" %}
14908 
14909   ins_cost(INSN_COST);
14910   ins_encode %{
14911     __ andr(as_Register($dst$$reg),
14912             as_Register($src1$$reg),
14913             as_Register($src2$$reg));
14914   %}
14915 
14916   ins_pipe(ialu_reg_reg);
14917 %}
14918 
14919 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14920   match(Set dst (AndL src1 src2));
14921 
14922   format %{ "and  $dst, $src1, $src2\t# int" %}
14923 
14924   ins_cost(INSN_COST);
14925   ins_encode %{
14926     __ andr(as_Register($dst$$reg),
14927             as_Register($src1$$reg),
14928             (uint64_t)($src2$$constant));
14929   %}
14930 
14931   ins_pipe(ialu_reg_imm);
14932 %}
14933 
14934 // Or Instructions
14935 
14936 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14937   match(Set dst (OrL src1 src2));
14938 
14939   format %{ "orr  $dst, $src1, $src2\t# int" %}
14940 
14941   ins_cost(INSN_COST);
14942   ins_encode %{
14943     __ orr(as_Register($dst$$reg),
14944            as_Register($src1$$reg),
14945            as_Register($src2$$reg));
14946   %}
14947 
14948   ins_pipe(ialu_reg_reg);
14949 %}
14950 
14951 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14952   match(Set dst (OrL src1 src2));
14953 
14954   format %{ "orr  $dst, $src1, $src2\t# int" %}
14955 
14956   ins_cost(INSN_COST);
14957   ins_encode %{
14958     __ orr(as_Register($dst$$reg),
14959            as_Register($src1$$reg),
14960            (uint64_t)($src2$$constant));
14961   %}
14962 
14963   ins_pipe(ialu_reg_imm);
14964 %}
14965 
14966 // Xor Instructions
14967 
14968 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14969   match(Set dst (XorL src1 src2));
14970 
14971   format %{ "eor  $dst, $src1, $src2\t# int" %}
14972 
14973   ins_cost(INSN_COST);
14974   ins_encode %{
14975     __ eor(as_Register($dst$$reg),
14976            as_Register($src1$$reg),
14977            as_Register($src2$$reg));
14978   %}
14979 
14980   ins_pipe(ialu_reg_reg);
14981 %}
14982 
14983 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14984   match(Set dst (XorL src1 src2));
14985 
14986   ins_cost(INSN_COST);
14987   format %{ "eor  $dst, $src1, $src2\t# int" %}
14988 
14989   ins_encode %{
14990     __ eor(as_Register($dst$$reg),
14991            as_Register($src1$$reg),
14992            (uint64_t)($src2$$constant));
14993   %}
14994 
14995   ins_pipe(ialu_reg_imm);
14996 %}
14997 
14998 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14999 %{
15000   match(Set dst (ConvI2L src));
15001 
15002   ins_cost(INSN_COST);
15003   format %{ "sxtw  $dst, $src\t# i2l" %}
15004   ins_encode %{
15005     __ sbfm($dst$$Register, $src$$Register, 0, 31);
15006   %}
15007   ins_pipe(ialu_reg_shift);
15008 %}
15009 
15010 // this pattern occurs in bigmath arithmetic
15011 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
15012 %{
15013   match(Set dst (AndL (ConvI2L src) mask));
15014 
15015   ins_cost(INSN_COST);
15016   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
15017   ins_encode %{
15018     __ ubfm($dst$$Register, $src$$Register, 0, 31);
15019   %}
15020 
15021   ins_pipe(ialu_reg_shift);
15022 %}
15023 
15024 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15025   match(Set dst (ConvL2I src));
15026 
15027   ins_cost(INSN_COST);
15028   format %{ "movw  $dst, $src \t// l2i" %}
15029 
15030   ins_encode %{
15031     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15032   %}
15033 
15034   ins_pipe(ialu_reg);
15035 %}
15036 
15037 instruct convD2F_reg(vRegF dst, vRegD src) %{
15038   match(Set dst (ConvD2F src));
15039 
15040   ins_cost(INSN_COST * 5);
15041   format %{ "fcvtd  $dst, $src \t// d2f" %}
15042 
15043   ins_encode %{
15044     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15045   %}
15046 
15047   ins_pipe(fp_d2f);
15048 %}
15049 
15050 instruct convF2D_reg(vRegD dst, vRegF src) %{
15051   match(Set dst (ConvF2D src));
15052 
15053   ins_cost(INSN_COST * 5);
15054   format %{ "fcvts  $dst, $src \t// f2d" %}
15055 
15056   ins_encode %{
15057     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15058   %}
15059 
15060   ins_pipe(fp_f2d);
15061 %}
15062 
15063 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15064   match(Set dst (ConvF2I src));
15065 
15066   ins_cost(INSN_COST * 5);
15067   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
15068 
15069   ins_encode %{
15070     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15071   %}
15072 
15073   ins_pipe(fp_f2i);
15074 %}
15075 
15076 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15077   match(Set dst (ConvF2L src));
15078 
15079   ins_cost(INSN_COST * 5);
15080   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15081 
15082   ins_encode %{
15083     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15084   %}
15085 
15086   ins_pipe(fp_f2l);
15087 %}
15088 
15089 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15090   match(Set dst (ConvF2HF src));
15091   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15092             "smov $dst, $tmp\t# move result from $tmp to $dst"
15093   %}
15094   effect(TEMP tmp);
15095   ins_encode %{
15096       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
15097   %}
15098   ins_pipe(pipe_slow);
15099 %}
15100 
15101 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15102   match(Set dst (ConvHF2F src));
15103   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15104             "fcvt $dst, $tmp\t# convert half to single precision"
15105   %}
15106   effect(TEMP tmp);
15107   ins_encode %{
15108       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
15109   %}
15110   ins_pipe(pipe_slow);
15111 %}
15112 
15113 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15114   match(Set dst (ConvI2F src));
15115 
15116   ins_cost(INSN_COST * 5);
15117   format %{ "scvtfws  $dst, $src \t// i2f" %}
15118 
15119   ins_encode %{
15120     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15121   %}
15122 
15123   ins_pipe(fp_i2f);
15124 %}
15125 
15126 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15127   match(Set dst (ConvL2F src));
15128 
15129   ins_cost(INSN_COST * 5);
15130   format %{ "scvtfs  $dst, $src \t// l2f" %}
15131 
15132   ins_encode %{
15133     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15134   %}
15135 
15136   ins_pipe(fp_l2f);
15137 %}
15138 
15139 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15140   match(Set dst (ConvD2I src));
15141 
15142   ins_cost(INSN_COST * 5);
15143   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15144 
15145   ins_encode %{
15146     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15147   %}
15148 
15149   ins_pipe(fp_d2i);
15150 %}
15151 
15152 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15153   match(Set dst (ConvD2L src));
15154 
15155   ins_cost(INSN_COST * 5);
15156   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15157 
15158   ins_encode %{
15159     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15160   %}
15161 
15162   ins_pipe(fp_d2l);
15163 %}
15164 
15165 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15166   match(Set dst (ConvI2D src));
15167 
15168   ins_cost(INSN_COST * 5);
15169   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15170 
15171   ins_encode %{
15172     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15173   %}
15174 
15175   ins_pipe(fp_i2d);
15176 %}
15177 
15178 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15179   match(Set dst (ConvL2D src));
15180 
15181   ins_cost(INSN_COST * 5);
15182   format %{ "scvtfd  $dst, $src \t// l2d" %}
15183 
15184   ins_encode %{
15185     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15186   %}
15187 
15188   ins_pipe(fp_l2d);
15189 %}
15190 
15191 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15192 %{
15193   match(Set dst (RoundD src));
15194   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15195   format %{ "java_round_double $dst,$src"%}
15196   ins_encode %{
15197     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15198                          as_FloatRegister($ftmp$$reg));
15199   %}
15200   ins_pipe(pipe_slow);
15201 %}
15202 
15203 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15204 %{
15205   match(Set dst (RoundF src));
15206   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15207   format %{ "java_round_float $dst,$src"%}
15208   ins_encode %{
15209     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15210                         as_FloatRegister($ftmp$$reg));
15211   %}
15212   ins_pipe(pipe_slow);
15213 %}
15214 
15215 // stack <-> reg and reg <-> reg shuffles with no conversion
15216 
15217 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15218 
15219   match(Set dst (MoveF2I src));
15220 
15221   effect(DEF dst, USE src);
15222 
15223   ins_cost(4 * INSN_COST);
15224 
15225   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15226 
15227   ins_encode %{
15228     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15229   %}
15230 
15231   ins_pipe(iload_reg_reg);
15232 
15233 %}
15234 
15235 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15236 
15237   match(Set dst (MoveI2F src));
15238 
15239   effect(DEF dst, USE src);
15240 
15241   ins_cost(4 * INSN_COST);
15242 
15243   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15244 
15245   ins_encode %{
15246     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15247   %}
15248 
15249   ins_pipe(pipe_class_memory);
15250 
15251 %}
15252 
15253 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15254 
15255   match(Set dst (MoveD2L src));
15256 
15257   effect(DEF dst, USE src);
15258 
15259   ins_cost(4 * INSN_COST);
15260 
15261   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15262 
15263   ins_encode %{
15264     __ ldr($dst$$Register, Address(sp, $src$$disp));
15265   %}
15266 
15267   ins_pipe(iload_reg_reg);
15268 
15269 %}
15270 
15271 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15272 
15273   match(Set dst (MoveL2D src));
15274 
15275   effect(DEF dst, USE src);
15276 
15277   ins_cost(4 * INSN_COST);
15278 
15279   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15280 
15281   ins_encode %{
15282     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15283   %}
15284 
15285   ins_pipe(pipe_class_memory);
15286 
15287 %}
15288 
15289 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15290 
15291   match(Set dst (MoveF2I src));
15292 
15293   effect(DEF dst, USE src);
15294 
15295   ins_cost(INSN_COST);
15296 
15297   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15298 
15299   ins_encode %{
15300     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15301   %}
15302 
15303   ins_pipe(pipe_class_memory);
15304 
15305 %}
15306 
15307 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15308 
15309   match(Set dst (MoveI2F src));
15310 
15311   effect(DEF dst, USE src);
15312 
15313   ins_cost(INSN_COST);
15314 
15315   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15316 
15317   ins_encode %{
15318     __ strw($src$$Register, Address(sp, $dst$$disp));
15319   %}
15320 
15321   ins_pipe(istore_reg_reg);
15322 
15323 %}
15324 
15325 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15326 
15327   match(Set dst (MoveD2L src));
15328 
15329   effect(DEF dst, USE src);
15330 
15331   ins_cost(INSN_COST);
15332 
15333   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15334 
15335   ins_encode %{
15336     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15337   %}
15338 
15339   ins_pipe(pipe_class_memory);
15340 
15341 %}
15342 
15343 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15344 
15345   match(Set dst (MoveL2D src));
15346 
15347   effect(DEF dst, USE src);
15348 
15349   ins_cost(INSN_COST);
15350 
15351   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15352 
15353   ins_encode %{
15354     __ str($src$$Register, Address(sp, $dst$$disp));
15355   %}
15356 
15357   ins_pipe(istore_reg_reg);
15358 
15359 %}
15360 
15361 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15362 
15363   match(Set dst (MoveF2I src));
15364 
15365   effect(DEF dst, USE src);
15366 
15367   ins_cost(INSN_COST);
15368 
15369   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15370 
15371   ins_encode %{
15372     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15373   %}
15374 
15375   ins_pipe(fp_f2i);
15376 
15377 %}
15378 
15379 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15380 
15381   match(Set dst (MoveI2F src));
15382 
15383   effect(DEF dst, USE src);
15384 
15385   ins_cost(INSN_COST);
15386 
15387   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15388 
15389   ins_encode %{
15390     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15391   %}
15392 
15393   ins_pipe(fp_i2f);
15394 
15395 %}
15396 
15397 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15398 
15399   match(Set dst (MoveD2L src));
15400 
15401   effect(DEF dst, USE src);
15402 
15403   ins_cost(INSN_COST);
15404 
15405   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15406 
15407   ins_encode %{
15408     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15409   %}
15410 
15411   ins_pipe(fp_d2l);
15412 
15413 %}
15414 
15415 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15416 
15417   match(Set dst (MoveL2D src));
15418 
15419   effect(DEF dst, USE src);
15420 
15421   ins_cost(INSN_COST);
15422 
15423   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15424 
15425   ins_encode %{
15426     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15427   %}
15428 
15429   ins_pipe(fp_l2d);
15430 
15431 %}
15432 
15433 // ============================================================================
15434 // clearing of an array
15435 
15436 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15437 %{
15438   match(Set dummy (ClearArray cnt base));
15439   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15440 
15441   ins_cost(4 * INSN_COST);
15442   format %{ "ClearArray $cnt, $base" %}
15443 
15444   ins_encode %{
15445     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15446     if (tpc == NULL) {
15447       ciEnv::current()->record_failure("CodeCache is full");
15448       return;
15449     }
15450   %}
15451 
15452   ins_pipe(pipe_class_memory);
15453 %}
15454 
15455 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15456 %{
15457   predicate((uint64_t)n->in(2)->get_long()
15458             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15459   match(Set dummy (ClearArray cnt base));
15460   effect(TEMP temp, USE_KILL base, KILL cr);
15461 
15462   ins_cost(4 * INSN_COST);
15463   format %{ "ClearArray $cnt, $base" %}
15464 
15465   ins_encode %{
15466     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15467     if (tpc == NULL) {
15468       ciEnv::current()->record_failure("CodeCache is full");
15469       return;
15470     }
15471   %}
15472 
15473   ins_pipe(pipe_class_memory);
15474 %}
15475 
15476 // ============================================================================
15477 // Overflow Math Instructions
15478 
15479 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15480 %{
15481   match(Set cr (OverflowAddI op1 op2));
15482 
15483   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15484   ins_cost(INSN_COST);
15485   ins_encode %{
15486     __ cmnw($op1$$Register, $op2$$Register);
15487   %}
15488 
15489   ins_pipe(icmp_reg_reg);
15490 %}
15491 
15492 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15493 %{
15494   match(Set cr (OverflowAddI op1 op2));
15495 
15496   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15497   ins_cost(INSN_COST);
15498   ins_encode %{
15499     __ cmnw($op1$$Register, $op2$$constant);
15500   %}
15501 
15502   ins_pipe(icmp_reg_imm);
15503 %}
15504 
15505 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15506 %{
15507   match(Set cr (OverflowAddL op1 op2));
15508 
15509   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15510   ins_cost(INSN_COST);
15511   ins_encode %{
15512     __ cmn($op1$$Register, $op2$$Register);
15513   %}
15514 
15515   ins_pipe(icmp_reg_reg);
15516 %}
15517 
15518 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15519 %{
15520   match(Set cr (OverflowAddL op1 op2));
15521 
15522   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15523   ins_cost(INSN_COST);
15524   ins_encode %{
15525     __ adds(zr, $op1$$Register, $op2$$constant);
15526   %}
15527 
15528   ins_pipe(icmp_reg_imm);
15529 %}
15530 
15531 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15532 %{
15533   match(Set cr (OverflowSubI op1 op2));
15534 
15535   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15536   ins_cost(INSN_COST);
15537   ins_encode %{
15538     __ cmpw($op1$$Register, $op2$$Register);
15539   %}
15540 
15541   ins_pipe(icmp_reg_reg);
15542 %}
15543 
15544 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15545 %{
15546   match(Set cr (OverflowSubI op1 op2));
15547 
15548   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15549   ins_cost(INSN_COST);
15550   ins_encode %{
15551     __ cmpw($op1$$Register, $op2$$constant);
15552   %}
15553 
15554   ins_pipe(icmp_reg_imm);
15555 %}
15556 
15557 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15558 %{
15559   match(Set cr (OverflowSubL op1 op2));
15560 
15561   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15562   ins_cost(INSN_COST);
15563   ins_encode %{
15564     __ cmp($op1$$Register, $op2$$Register);
15565   %}
15566 
15567   ins_pipe(icmp_reg_reg);
15568 %}
15569 
15570 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15571 %{
15572   match(Set cr (OverflowSubL op1 op2));
15573 
15574   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15575   ins_cost(INSN_COST);
15576   ins_encode %{
15577     __ subs(zr, $op1$$Register, $op2$$constant);
15578   %}
15579 
15580   ins_pipe(icmp_reg_imm);
15581 %}
15582 
15583 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15584 %{
15585   match(Set cr (OverflowSubI zero op1));
15586 
15587   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15588   ins_cost(INSN_COST);
15589   ins_encode %{
15590     __ cmpw(zr, $op1$$Register);
15591   %}
15592 
15593   ins_pipe(icmp_reg_imm);
15594 %}
15595 
15596 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15597 %{
15598   match(Set cr (OverflowSubL zero op1));
15599 
15600   format %{ "cmp   zr, $op1\t# overflow check long" %}
15601   ins_cost(INSN_COST);
15602   ins_encode %{
15603     __ cmp(zr, $op1$$Register);
15604   %}
15605 
15606   ins_pipe(icmp_reg_imm);
15607 %}
15608 
15609 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15610 %{
15611   match(Set cr (OverflowMulI op1 op2));
15612 
15613   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15614             "cmp   rscratch1, rscratch1, sxtw\n\t"
15615             "movw  rscratch1, #0x80000000\n\t"
15616             "cselw rscratch1, rscratch1, zr, NE\n\t"
15617             "cmpw  rscratch1, #1" %}
15618   ins_cost(5 * INSN_COST);
15619   ins_encode %{
15620     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15621     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15622     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15623     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15624     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15625   %}
15626 
15627   ins_pipe(pipe_slow);
15628 %}
15629 
15630 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15631 %{
15632   match(If cmp (OverflowMulI op1 op2));
15633   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15634             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15635   effect(USE labl, KILL cr);
15636 
15637   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15638             "cmp   rscratch1, rscratch1, sxtw\n\t"
15639             "b$cmp   $labl" %}
15640   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15641   ins_encode %{
15642     Label* L = $labl$$label;
15643     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15644     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15645     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15646     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15647   %}
15648 
15649   ins_pipe(pipe_serial);
15650 %}
15651 
15652 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15653 %{
15654   match(Set cr (OverflowMulL op1 op2));
15655 
15656   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15657             "smulh rscratch2, $op1, $op2\n\t"
15658             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15659             "movw  rscratch1, #0x80000000\n\t"
15660             "cselw rscratch1, rscratch1, zr, NE\n\t"
15661             "cmpw  rscratch1, #1" %}
15662   ins_cost(6 * INSN_COST);
15663   ins_encode %{
15664     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15665     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15666     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15667     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15668     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15669     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15670   %}
15671 
15672   ins_pipe(pipe_slow);
15673 %}
15674 
15675 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15676 %{
15677   match(If cmp (OverflowMulL op1 op2));
15678   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15679             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15680   effect(USE labl, KILL cr);
15681 
15682   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15683             "smulh rscratch2, $op1, $op2\n\t"
15684             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15685             "b$cmp $labl" %}
15686   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15687   ins_encode %{
15688     Label* L = $labl$$label;
15689     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15690     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15691     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15692     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15693     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15694   %}
15695 
15696   ins_pipe(pipe_serial);
15697 %}
15698 
15699 // ============================================================================
15700 // Compare Instructions
15701 
15702 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15703 %{
15704   match(Set cr (CmpI op1 op2));
15705 
15706   effect(DEF cr, USE op1, USE op2);
15707 
15708   ins_cost(INSN_COST);
15709   format %{ "cmpw  $op1, $op2" %}
15710 
15711   ins_encode(aarch64_enc_cmpw(op1, op2));
15712 
15713   ins_pipe(icmp_reg_reg);
15714 %}
15715 
15716 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15717 %{
15718   match(Set cr (CmpI op1 zero));
15719 
15720   effect(DEF cr, USE op1);
15721 
15722   ins_cost(INSN_COST);
15723   format %{ "cmpw $op1, 0" %}
15724 
15725   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15726 
15727   ins_pipe(icmp_reg_imm);
15728 %}
15729 
15730 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15731 %{
15732   match(Set cr (CmpI op1 op2));
15733 
15734   effect(DEF cr, USE op1);
15735 
15736   ins_cost(INSN_COST);
15737   format %{ "cmpw  $op1, $op2" %}
15738 
15739   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15740 
15741   ins_pipe(icmp_reg_imm);
15742 %}
15743 
15744 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15745 %{
15746   match(Set cr (CmpI op1 op2));
15747 
15748   effect(DEF cr, USE op1);
15749 
15750   ins_cost(INSN_COST * 2);
15751   format %{ "cmpw  $op1, $op2" %}
15752 
15753   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15754 
15755   ins_pipe(icmp_reg_imm);
15756 %}
15757 
15758 // Unsigned compare Instructions; really, same as signed compare
15759 // except it should only be used to feed an If or a CMovI which takes a
15760 // cmpOpU.
15761 
15762 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15763 %{
15764   match(Set cr (CmpU op1 op2));
15765 
15766   effect(DEF cr, USE op1, USE op2);
15767 
15768   ins_cost(INSN_COST);
15769   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15770 
15771   ins_encode(aarch64_enc_cmpw(op1, op2));
15772 
15773   ins_pipe(icmp_reg_reg);
15774 %}
15775 
15776 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15777 %{
15778   match(Set cr (CmpU op1 zero));
15779 
15780   effect(DEF cr, USE op1);
15781 
15782   ins_cost(INSN_COST);
15783   format %{ "cmpw $op1, #0\t# unsigned" %}
15784 
15785   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15786 
15787   ins_pipe(icmp_reg_imm);
15788 %}
15789 
15790 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15791 %{
15792   match(Set cr (CmpU op1 op2));
15793 
15794   effect(DEF cr, USE op1);
15795 
15796   ins_cost(INSN_COST);
15797   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15798 
15799   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15800 
15801   ins_pipe(icmp_reg_imm);
15802 %}
15803 
15804 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15805 %{
15806   match(Set cr (CmpU op1 op2));
15807 
15808   effect(DEF cr, USE op1);
15809 
15810   ins_cost(INSN_COST * 2);
15811   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15812 
15813   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15814 
15815   ins_pipe(icmp_reg_imm);
15816 %}
15817 
15818 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15819 %{
15820   match(Set cr (CmpL op1 op2));
15821 
15822   effect(DEF cr, USE op1, USE op2);
15823 
15824   ins_cost(INSN_COST);
15825   format %{ "cmp  $op1, $op2" %}
15826 
15827   ins_encode(aarch64_enc_cmp(op1, op2));
15828 
15829   ins_pipe(icmp_reg_reg);
15830 %}
15831 
15832 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15833 %{
15834   match(Set cr (CmpL op1 zero));
15835 
15836   effect(DEF cr, USE op1);
15837 
15838   ins_cost(INSN_COST);
15839   format %{ "tst  $op1" %}
15840 
15841   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15842 
15843   ins_pipe(icmp_reg_imm);
15844 %}
15845 
15846 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15847 %{
15848   match(Set cr (CmpL op1 op2));
15849 
15850   effect(DEF cr, USE op1);
15851 
15852   ins_cost(INSN_COST);
15853   format %{ "cmp  $op1, $op2" %}
15854 
15855   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15856 
15857   ins_pipe(icmp_reg_imm);
15858 %}
15859 
15860 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15861 %{
15862   match(Set cr (CmpL op1 op2));
15863 
15864   effect(DEF cr, USE op1);
15865 
15866   ins_cost(INSN_COST * 2);
15867   format %{ "cmp  $op1, $op2" %}
15868 
15869   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15870 
15871   ins_pipe(icmp_reg_imm);
15872 %}
15873 
15874 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15875 %{
15876   match(Set cr (CmpUL op1 op2));
15877 
15878   effect(DEF cr, USE op1, USE op2);
15879 
15880   ins_cost(INSN_COST);
15881   format %{ "cmp  $op1, $op2" %}
15882 
15883   ins_encode(aarch64_enc_cmp(op1, op2));
15884 
15885   ins_pipe(icmp_reg_reg);
15886 %}
15887 
15888 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15889 %{
15890   match(Set cr (CmpUL op1 zero));
15891 
15892   effect(DEF cr, USE op1);
15893 
15894   ins_cost(INSN_COST);
15895   format %{ "tst  $op1" %}
15896 
15897   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15898 
15899   ins_pipe(icmp_reg_imm);
15900 %}
15901 
15902 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15903 %{
15904   match(Set cr (CmpUL op1 op2));
15905 
15906   effect(DEF cr, USE op1);
15907 
15908   ins_cost(INSN_COST);
15909   format %{ "cmp  $op1, $op2" %}
15910 
15911   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15912 
15913   ins_pipe(icmp_reg_imm);
15914 %}
15915 
15916 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15917 %{
15918   match(Set cr (CmpUL op1 op2));
15919 
15920   effect(DEF cr, USE op1);
15921 
15922   ins_cost(INSN_COST * 2);
15923   format %{ "cmp  $op1, $op2" %}
15924 
15925   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15926 
15927   ins_pipe(icmp_reg_imm);
15928 %}
15929 
15930 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15931 %{
15932   match(Set cr (CmpP op1 op2));
15933 
15934   effect(DEF cr, USE op1, USE op2);
15935 
15936   ins_cost(INSN_COST);
15937   format %{ "cmp  $op1, $op2\t // ptr" %}
15938 
15939   ins_encode(aarch64_enc_cmpp(op1, op2));
15940 
15941   ins_pipe(icmp_reg_reg);
15942 %}
15943 
15944 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15945 %{
15946   match(Set cr (CmpN op1 op2));
15947 
15948   effect(DEF cr, USE op1, USE op2);
15949 
15950   ins_cost(INSN_COST);
15951   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15952 
15953   ins_encode(aarch64_enc_cmpn(op1, op2));
15954 
15955   ins_pipe(icmp_reg_reg);
15956 %}
15957 
15958 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15959 %{
15960   match(Set cr (CmpP op1 zero));
15961 
15962   effect(DEF cr, USE op1, USE zero);
15963 
15964   ins_cost(INSN_COST);
15965   format %{ "cmp  $op1, 0\t // ptr" %}
15966 
15967   ins_encode(aarch64_enc_testp(op1));
15968 
15969   ins_pipe(icmp_reg_imm);
15970 %}
15971 
15972 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15973 %{
15974   match(Set cr (CmpN op1 zero));
15975 
15976   effect(DEF cr, USE op1, USE zero);
15977 
15978   ins_cost(INSN_COST);
15979   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15980 
15981   ins_encode(aarch64_enc_testn(op1));
15982 
15983   ins_pipe(icmp_reg_imm);
15984 %}
15985 
15986 // FP comparisons
15987 //
15988 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15989 // using normal cmpOp. See declaration of rFlagsReg for details.
15990 
15991 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15992 %{
15993   match(Set cr (CmpF src1 src2));
15994 
15995   ins_cost(3 * INSN_COST);
15996   format %{ "fcmps $src1, $src2" %}
15997 
15998   ins_encode %{
15999     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16000   %}
16001 
16002   ins_pipe(pipe_class_compare);
16003 %}
16004 
16005 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
16006 %{
16007   match(Set cr (CmpF src1 src2));
16008 
16009   ins_cost(3 * INSN_COST);
16010   format %{ "fcmps $src1, 0.0" %}
16011 
16012   ins_encode %{
16013     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
16014   %}
16015 
16016   ins_pipe(pipe_class_compare);
16017 %}
16018 // FROM HERE
16019 
16020 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16021 %{
16022   match(Set cr (CmpD src1 src2));
16023 
16024   ins_cost(3 * INSN_COST);
16025   format %{ "fcmpd $src1, $src2" %}
16026 
16027   ins_encode %{
16028     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16029   %}
16030 
16031   ins_pipe(pipe_class_compare);
16032 %}
16033 
16034 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16035 %{
16036   match(Set cr (CmpD src1 src2));
16037 
16038   ins_cost(3 * INSN_COST);
16039   format %{ "fcmpd $src1, 0.0" %}
16040 
16041   ins_encode %{
16042     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16043   %}
16044 
16045   ins_pipe(pipe_class_compare);
16046 %}
16047 
16048 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16049 %{
16050   match(Set dst (CmpF3 src1 src2));
16051   effect(KILL cr);
16052 
16053   ins_cost(5 * INSN_COST);
16054   format %{ "fcmps $src1, $src2\n\t"
16055             "csinvw($dst, zr, zr, eq\n\t"
16056             "csnegw($dst, $dst, $dst, lt)"
16057   %}
16058 
16059   ins_encode %{
16060     Label done;
16061     FloatRegister s1 = as_FloatRegister($src1$$reg);
16062     FloatRegister s2 = as_FloatRegister($src2$$reg);
16063     Register d = as_Register($dst$$reg);
16064     __ fcmps(s1, s2);
16065     // installs 0 if EQ else -1
16066     __ csinvw(d, zr, zr, Assembler::EQ);
16067     // keeps -1 if less or unordered else installs 1
16068     __ csnegw(d, d, d, Assembler::LT);
16069     __ bind(done);
16070   %}
16071 
16072   ins_pipe(pipe_class_default);
16073 
16074 %}
16075 
16076 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16077 %{
16078   match(Set dst (CmpD3 src1 src2));
16079   effect(KILL cr);
16080 
16081   ins_cost(5 * INSN_COST);
16082   format %{ "fcmpd $src1, $src2\n\t"
16083             "csinvw($dst, zr, zr, eq\n\t"
16084             "csnegw($dst, $dst, $dst, lt)"
16085   %}
16086 
16087   ins_encode %{
16088     Label done;
16089     FloatRegister s1 = as_FloatRegister($src1$$reg);
16090     FloatRegister s2 = as_FloatRegister($src2$$reg);
16091     Register d = as_Register($dst$$reg);
16092     __ fcmpd(s1, s2);
16093     // installs 0 if EQ else -1
16094     __ csinvw(d, zr, zr, Assembler::EQ);
16095     // keeps -1 if less or unordered else installs 1
16096     __ csnegw(d, d, d, Assembler::LT);
16097     __ bind(done);
16098   %}
16099   ins_pipe(pipe_class_default);
16100 
16101 %}
16102 
16103 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16104 %{
16105   match(Set dst (CmpF3 src1 zero));
16106   effect(KILL cr);
16107 
16108   ins_cost(5 * INSN_COST);
16109   format %{ "fcmps $src1, 0.0\n\t"
16110             "csinvw($dst, zr, zr, eq\n\t"
16111             "csnegw($dst, $dst, $dst, lt)"
16112   %}
16113 
16114   ins_encode %{
16115     Label done;
16116     FloatRegister s1 = as_FloatRegister($src1$$reg);
16117     Register d = as_Register($dst$$reg);
16118     __ fcmps(s1, 0.0);
16119     // installs 0 if EQ else -1
16120     __ csinvw(d, zr, zr, Assembler::EQ);
16121     // keeps -1 if less or unordered else installs 1
16122     __ csnegw(d, d, d, Assembler::LT);
16123     __ bind(done);
16124   %}
16125 
16126   ins_pipe(pipe_class_default);
16127 
16128 %}
16129 
16130 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16131 %{
16132   match(Set dst (CmpD3 src1 zero));
16133   effect(KILL cr);
16134 
16135   ins_cost(5 * INSN_COST);
16136   format %{ "fcmpd $src1, 0.0\n\t"
16137             "csinvw($dst, zr, zr, eq\n\t"
16138             "csnegw($dst, $dst, $dst, lt)"
16139   %}
16140 
16141   ins_encode %{
16142     Label done;
16143     FloatRegister s1 = as_FloatRegister($src1$$reg);
16144     Register d = as_Register($dst$$reg);
16145     __ fcmpd(s1, 0.0);
16146     // installs 0 if EQ else -1
16147     __ csinvw(d, zr, zr, Assembler::EQ);
16148     // keeps -1 if less or unordered else installs 1
16149     __ csnegw(d, d, d, Assembler::LT);
16150     __ bind(done);
16151   %}
16152   ins_pipe(pipe_class_default);
16153 
16154 %}
16155 
16156 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16157 %{
16158   match(Set dst (CmpLTMask p q));
16159   effect(KILL cr);
16160 
16161   ins_cost(3 * INSN_COST);
16162 
16163   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16164             "csetw $dst, lt\n\t"
16165             "subw $dst, zr, $dst"
16166   %}
16167 
16168   ins_encode %{
16169     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16170     __ csetw(as_Register($dst$$reg), Assembler::LT);
16171     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16172   %}
16173 
16174   ins_pipe(ialu_reg_reg);
16175 %}
16176 
16177 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16178 %{
16179   match(Set dst (CmpLTMask src zero));
16180   effect(KILL cr);
16181 
16182   ins_cost(INSN_COST);
16183 
16184   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16185 
16186   ins_encode %{
16187     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16188   %}
16189 
16190   ins_pipe(ialu_reg_shift);
16191 %}
16192 
16193 // ============================================================================
16194 // Max and Min
16195 
16196 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16197 
16198 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16199 %{
16200   effect(DEF cr, USE src);
16201   ins_cost(INSN_COST);
16202   format %{ "cmpw $src, 0" %}
16203 
16204   ins_encode %{
16205     __ cmpw($src$$Register, 0);
16206   %}
16207   ins_pipe(icmp_reg_imm);
16208 %}
16209 
16210 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16211 %{
16212   match(Set dst (MinI src1 src2));
16213   ins_cost(INSN_COST * 3);
16214 
16215   expand %{
16216     rFlagsReg cr;
16217     compI_reg_reg(cr, src1, src2);
16218     cmovI_reg_reg_lt(dst, src1, src2, cr);
16219   %}
16220 %}
16221 
16222 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16223 %{
16224   match(Set dst (MaxI src1 src2));
16225   ins_cost(INSN_COST * 3);
16226 
16227   expand %{
16228     rFlagsReg cr;
16229     compI_reg_reg(cr, src1, src2);
16230     cmovI_reg_reg_gt(dst, src1, src2, cr);
16231   %}
16232 %}
16233 
16234 
16235 // ============================================================================
16236 // Branch Instructions
16237 
16238 // Direct Branch.
16239 instruct branch(label lbl)
16240 %{
16241   match(Goto);
16242 
16243   effect(USE lbl);
16244 
16245   ins_cost(BRANCH_COST);
16246   format %{ "b  $lbl" %}
16247 
16248   ins_encode(aarch64_enc_b(lbl));
16249 
16250   ins_pipe(pipe_branch);
16251 %}
16252 
16253 // Conditional Near Branch
16254 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16255 %{
16256   // Same match rule as `branchConFar'.
16257   match(If cmp cr);
16258 
16259   effect(USE lbl);
16260 
16261   ins_cost(BRANCH_COST);
16262   // If set to 1 this indicates that the current instruction is a
16263   // short variant of a long branch. This avoids using this
16264   // instruction in first-pass matching. It will then only be used in
16265   // the `Shorten_branches' pass.
16266   // ins_short_branch(1);
16267   format %{ "b$cmp  $lbl" %}
16268 
16269   ins_encode(aarch64_enc_br_con(cmp, lbl));
16270 
16271   ins_pipe(pipe_branch_cond);
16272 %}
16273 
16274 // Conditional Near Branch Unsigned
16275 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16276 %{
16277   // Same match rule as `branchConFar'.
16278   match(If cmp cr);
16279 
16280   effect(USE lbl);
16281 
16282   ins_cost(BRANCH_COST);
16283   // If set to 1 this indicates that the current instruction is a
16284   // short variant of a long branch. This avoids using this
16285   // instruction in first-pass matching. It will then only be used in
16286   // the `Shorten_branches' pass.
16287   // ins_short_branch(1);
16288   format %{ "b$cmp  $lbl\t# unsigned" %}
16289 
16290   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16291 
16292   ins_pipe(pipe_branch_cond);
16293 %}
16294 
16295 // Make use of CBZ and CBNZ.  These instructions, as well as being
16296 // shorter than (cmp; branch), have the additional benefit of not
16297 // killing the flags.
16298 
16299 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16300   match(If cmp (CmpI op1 op2));
16301   effect(USE labl);
16302 
16303   ins_cost(BRANCH_COST);
16304   format %{ "cbw$cmp   $op1, $labl" %}
16305   ins_encode %{
16306     Label* L = $labl$$label;
16307     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16308     if (cond == Assembler::EQ)
16309       __ cbzw($op1$$Register, *L);
16310     else
16311       __ cbnzw($op1$$Register, *L);
16312   %}
16313   ins_pipe(pipe_cmp_branch);
16314 %}
16315 
16316 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16317   match(If cmp (CmpL op1 op2));
16318   effect(USE labl);
16319 
16320   ins_cost(BRANCH_COST);
16321   format %{ "cb$cmp   $op1, $labl" %}
16322   ins_encode %{
16323     Label* L = $labl$$label;
16324     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16325     if (cond == Assembler::EQ)
16326       __ cbz($op1$$Register, *L);
16327     else
16328       __ cbnz($op1$$Register, *L);
16329   %}
16330   ins_pipe(pipe_cmp_branch);
16331 %}
16332 
16333 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16334   match(If cmp (CmpP op1 op2));
16335   effect(USE labl);
16336 
16337   ins_cost(BRANCH_COST);
16338   format %{ "cb$cmp   $op1, $labl" %}
16339   ins_encode %{
16340     Label* L = $labl$$label;
16341     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16342     if (cond == Assembler::EQ)
16343       __ cbz($op1$$Register, *L);
16344     else
16345       __ cbnz($op1$$Register, *L);
16346   %}
16347   ins_pipe(pipe_cmp_branch);
16348 %}
16349 
16350 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16351   match(If cmp (CmpN op1 op2));
16352   effect(USE labl);
16353 
16354   ins_cost(BRANCH_COST);
16355   format %{ "cbw$cmp   $op1, $labl" %}
16356   ins_encode %{
16357     Label* L = $labl$$label;
16358     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16359     if (cond == Assembler::EQ)
16360       __ cbzw($op1$$Register, *L);
16361     else
16362       __ cbnzw($op1$$Register, *L);
16363   %}
16364   ins_pipe(pipe_cmp_branch);
16365 %}
16366 
16367 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16368   match(If cmp (CmpP (DecodeN oop) zero));
16369   effect(USE labl);
16370 
16371   ins_cost(BRANCH_COST);
16372   format %{ "cb$cmp   $oop, $labl" %}
16373   ins_encode %{
16374     Label* L = $labl$$label;
16375     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16376     if (cond == Assembler::EQ)
16377       __ cbzw($oop$$Register, *L);
16378     else
16379       __ cbnzw($oop$$Register, *L);
16380   %}
16381   ins_pipe(pipe_cmp_branch);
16382 %}
16383 
16384 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16385   match(If cmp (CmpU op1 op2));
16386   effect(USE labl);
16387 
16388   ins_cost(BRANCH_COST);
16389   format %{ "cbw$cmp   $op1, $labl" %}
16390   ins_encode %{
16391     Label* L = $labl$$label;
16392     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16393     if (cond == Assembler::EQ || cond == Assembler::LS)
16394       __ cbzw($op1$$Register, *L);
16395     else
16396       __ cbnzw($op1$$Register, *L);
16397   %}
16398   ins_pipe(pipe_cmp_branch);
16399 %}
16400 
16401 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16402   match(If cmp (CmpUL op1 op2));
16403   effect(USE labl);
16404 
16405   ins_cost(BRANCH_COST);
16406   format %{ "cb$cmp   $op1, $labl" %}
16407   ins_encode %{
16408     Label* L = $labl$$label;
16409     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16410     if (cond == Assembler::EQ || cond == Assembler::LS)
16411       __ cbz($op1$$Register, *L);
16412     else
16413       __ cbnz($op1$$Register, *L);
16414   %}
16415   ins_pipe(pipe_cmp_branch);
16416 %}
16417 
16418 // Test bit and Branch
16419 
16420 // Patterns for short (< 32KiB) variants
16421 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16422   match(If cmp (CmpL op1 op2));
16423   effect(USE labl);
16424 
16425   ins_cost(BRANCH_COST);
16426   format %{ "cb$cmp   $op1, $labl # long" %}
16427   ins_encode %{
16428     Label* L = $labl$$label;
16429     Assembler::Condition cond =
16430       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16431     __ tbr(cond, $op1$$Register, 63, *L);
16432   %}
16433   ins_pipe(pipe_cmp_branch);
16434   ins_short_branch(1);
16435 %}
16436 
16437 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16438   match(If cmp (CmpI op1 op2));
16439   effect(USE labl);
16440 
16441   ins_cost(BRANCH_COST);
16442   format %{ "cb$cmp   $op1, $labl # int" %}
16443   ins_encode %{
16444     Label* L = $labl$$label;
16445     Assembler::Condition cond =
16446       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16447     __ tbr(cond, $op1$$Register, 31, *L);
16448   %}
16449   ins_pipe(pipe_cmp_branch);
16450   ins_short_branch(1);
16451 %}
16452 
16453 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16454   match(If cmp (CmpL (AndL op1 op2) op3));
16455   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16456   effect(USE labl);
16457 
16458   ins_cost(BRANCH_COST);
16459   format %{ "tb$cmp   $op1, $op2, $labl" %}
16460   ins_encode %{
16461     Label* L = $labl$$label;
16462     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16463     int bit = exact_log2_long($op2$$constant);
16464     __ tbr(cond, $op1$$Register, bit, *L);
16465   %}
16466   ins_pipe(pipe_cmp_branch);
16467   ins_short_branch(1);
16468 %}
16469 
16470 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16471   match(If cmp (CmpI (AndI op1 op2) op3));
16472   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16473   effect(USE labl);
16474 
16475   ins_cost(BRANCH_COST);
16476   format %{ "tb$cmp   $op1, $op2, $labl" %}
16477   ins_encode %{
16478     Label* L = $labl$$label;
16479     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16480     int bit = exact_log2((juint)$op2$$constant);
16481     __ tbr(cond, $op1$$Register, bit, *L);
16482   %}
16483   ins_pipe(pipe_cmp_branch);
16484   ins_short_branch(1);
16485 %}
16486 
16487 // And far variants
16488 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16489   match(If cmp (CmpL op1 op2));
16490   effect(USE labl);
16491 
16492   ins_cost(BRANCH_COST);
16493   format %{ "cb$cmp   $op1, $labl # long" %}
16494   ins_encode %{
16495     Label* L = $labl$$label;
16496     Assembler::Condition cond =
16497       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16498     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16499   %}
16500   ins_pipe(pipe_cmp_branch);
16501 %}
16502 
16503 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16504   match(If cmp (CmpI op1 op2));
16505   effect(USE labl);
16506 
16507   ins_cost(BRANCH_COST);
16508   format %{ "cb$cmp   $op1, $labl # int" %}
16509   ins_encode %{
16510     Label* L = $labl$$label;
16511     Assembler::Condition cond =
16512       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16513     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16514   %}
16515   ins_pipe(pipe_cmp_branch);
16516 %}
16517 
16518 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16519   match(If cmp (CmpL (AndL op1 op2) op3));
16520   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16521   effect(USE labl);
16522 
16523   ins_cost(BRANCH_COST);
16524   format %{ "tb$cmp   $op1, $op2, $labl" %}
16525   ins_encode %{
16526     Label* L = $labl$$label;
16527     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16528     int bit = exact_log2_long($op2$$constant);
16529     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16530   %}
16531   ins_pipe(pipe_cmp_branch);
16532 %}
16533 
16534 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16535   match(If cmp (CmpI (AndI op1 op2) op3));
16536   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16537   effect(USE labl);
16538 
16539   ins_cost(BRANCH_COST);
16540   format %{ "tb$cmp   $op1, $op2, $labl" %}
16541   ins_encode %{
16542     Label* L = $labl$$label;
16543     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16544     int bit = exact_log2((juint)$op2$$constant);
16545     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16546   %}
16547   ins_pipe(pipe_cmp_branch);
16548 %}
16549 
16550 // Test bits
16551 
16552 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16553   match(Set cr (CmpL (AndL op1 op2) op3));
16554   predicate(Assembler::operand_valid_for_logical_immediate
16555             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16556 
16557   ins_cost(INSN_COST);
16558   format %{ "tst $op1, $op2 # long" %}
16559   ins_encode %{
16560     __ tst($op1$$Register, $op2$$constant);
16561   %}
16562   ins_pipe(ialu_reg_reg);
16563 %}
16564 
16565 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16566   match(Set cr (CmpI (AndI op1 op2) op3));
16567   predicate(Assembler::operand_valid_for_logical_immediate
16568             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16569 
16570   ins_cost(INSN_COST);
16571   format %{ "tst $op1, $op2 # int" %}
16572   ins_encode %{
16573     __ tstw($op1$$Register, $op2$$constant);
16574   %}
16575   ins_pipe(ialu_reg_reg);
16576 %}
16577 
16578 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16579   match(Set cr (CmpL (AndL op1 op2) op3));
16580 
16581   ins_cost(INSN_COST);
16582   format %{ "tst $op1, $op2 # long" %}
16583   ins_encode %{
16584     __ tst($op1$$Register, $op2$$Register);
16585   %}
16586   ins_pipe(ialu_reg_reg);
16587 %}
16588 
16589 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16590   match(Set cr (CmpI (AndI op1 op2) op3));
16591 
16592   ins_cost(INSN_COST);
16593   format %{ "tstw $op1, $op2 # int" %}
16594   ins_encode %{
16595     __ tstw($op1$$Register, $op2$$Register);
16596   %}
16597   ins_pipe(ialu_reg_reg);
16598 %}
16599 
16600 
16601 // Conditional Far Branch
16602 // Conditional Far Branch Unsigned
16603 // TODO: fixme
16604 
16605 // counted loop end branch near
16606 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16607 %{
16608   match(CountedLoopEnd cmp cr);
16609 
16610   effect(USE lbl);
16611 
16612   ins_cost(BRANCH_COST);
16613   // short variant.
16614   // ins_short_branch(1);
16615   format %{ "b$cmp $lbl \t// counted loop end" %}
16616 
16617   ins_encode(aarch64_enc_br_con(cmp, lbl));
16618 
16619   ins_pipe(pipe_branch);
16620 %}
16621 
16622 // counted loop end branch far
16623 // TODO: fixme
16624 
16625 // ============================================================================
16626 // inlined locking and unlocking
16627 
16628 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16629 %{
16630   match(Set cr (FastLock object box));
16631   effect(TEMP tmp, TEMP tmp2);
16632 
16633   // TODO
16634   // identify correct cost
16635   ins_cost(5 * INSN_COST);
16636   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16637 
16638   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16639 
16640   ins_pipe(pipe_serial);
16641 %}
16642 
16643 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16644 %{
16645   match(Set cr (FastUnlock object box));
16646   effect(TEMP tmp, TEMP tmp2);
16647 
16648   ins_cost(5 * INSN_COST);
16649   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16650 
16651   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16652 
16653   ins_pipe(pipe_serial);
16654 %}
16655 
16656 
16657 // ============================================================================
16658 // Safepoint Instructions
16659 
16660 // TODO
16661 // provide a near and far version of this code
16662 
16663 instruct safePoint(rFlagsReg cr, iRegP poll)
16664 %{
16665   match(SafePoint poll);
16666   effect(KILL cr);
16667 
16668   format %{
16669     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16670   %}
16671   ins_encode %{
16672     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16673   %}
16674   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16675 %}
16676 
16677 
16678 // ============================================================================
16679 // Procedure Call/Return Instructions
16680 
16681 // Call Java Static Instruction
16682 
16683 instruct CallStaticJavaDirect(method meth)
16684 %{
16685   match(CallStaticJava);
16686 
16687   effect(USE meth);
16688 
16689   ins_cost(CALL_COST);
16690 
16691   format %{ "call,static $meth \t// ==> " %}
16692 
16693   ins_encode(aarch64_enc_java_static_call(meth),
16694              aarch64_enc_call_epilog);
16695 
16696   ins_pipe(pipe_class_call);
16697 %}
16698 
16699 // TO HERE
16700 
16701 // Call Java Dynamic Instruction
16702 instruct CallDynamicJavaDirect(method meth)
16703 %{
16704   match(CallDynamicJava);
16705 
16706   effect(USE meth);
16707 
16708   ins_cost(CALL_COST);
16709 
16710   format %{ "CALL,dynamic $meth \t// ==> " %}
16711 
16712   ins_encode(aarch64_enc_java_dynamic_call(meth),
16713              aarch64_enc_call_epilog);
16714 
16715   ins_pipe(pipe_class_call);
16716 %}
16717 
16718 // Call Runtime Instruction
16719 
16720 instruct CallRuntimeDirect(method meth)
16721 %{
16722   match(CallRuntime);
16723 
16724   effect(USE meth);
16725 
16726   ins_cost(CALL_COST);
16727 
16728   format %{ "CALL, runtime $meth" %}
16729 
16730   ins_encode( aarch64_enc_java_to_runtime(meth) );
16731 
16732   ins_pipe(pipe_class_call);
16733 %}
16734 
16735 // Call Runtime Instruction
16736 
16737 instruct CallLeafDirect(method meth)
16738 %{
16739   match(CallLeaf);
16740 
16741   effect(USE meth);
16742 
16743   ins_cost(CALL_COST);
16744 
16745   format %{ "CALL, runtime leaf $meth" %}
16746 
16747   ins_encode( aarch64_enc_java_to_runtime(meth) );
16748 
16749   ins_pipe(pipe_class_call);
16750 %}
16751 
16752 // Call Runtime Instruction
16753 
16754 instruct CallLeafNoFPDirect(method meth)
16755 %{
16756   match(CallLeafNoFP);
16757 
16758   effect(USE meth);
16759 
16760   ins_cost(CALL_COST);
16761 
16762   format %{ "CALL, runtime leaf nofp $meth" %}
16763 
16764   ins_encode( aarch64_enc_java_to_runtime(meth) );
16765 
16766   ins_pipe(pipe_class_call);
16767 %}
16768 
16769 // Tail Call; Jump from runtime stub to Java code.
16770 // Also known as an 'interprocedural jump'.
16771 // Target of jump will eventually return to caller.
16772 // TailJump below removes the return address.
16773 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16774 %{
16775   match(TailCall jump_target method_ptr);
16776 
16777   ins_cost(CALL_COST);
16778 
16779   format %{ "br $jump_target\t# $method_ptr holds method" %}
16780 
16781   ins_encode(aarch64_enc_tail_call(jump_target));
16782 
16783   ins_pipe(pipe_class_call);
16784 %}
16785 
16786 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16787 %{
16788   match(TailJump jump_target ex_oop);
16789 
16790   ins_cost(CALL_COST);
16791 
16792   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16793 
16794   ins_encode(aarch64_enc_tail_jmp(jump_target));
16795 
16796   ins_pipe(pipe_class_call);
16797 %}
16798 
16799 // Create exception oop: created by stack-crawling runtime code.
16800 // Created exception is now available to this handler, and is setup
16801 // just prior to jumping to this handler. No code emitted.
16802 // TODO check
16803 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16804 instruct CreateException(iRegP_R0 ex_oop)
16805 %{
16806   match(Set ex_oop (CreateEx));
16807 
16808   format %{ " -- \t// exception oop; no code emitted" %}
16809 
16810   size(0);
16811 
16812   ins_encode( /*empty*/ );
16813 
16814   ins_pipe(pipe_class_empty);
16815 %}
16816 
16817 // Rethrow exception: The exception oop will come in the first
16818 // argument position. Then JUMP (not call) to the rethrow stub code.
16819 instruct RethrowException() %{
16820   match(Rethrow);
16821   ins_cost(CALL_COST);
16822 
16823   format %{ "b rethrow_stub" %}
16824 
16825   ins_encode( aarch64_enc_rethrow() );
16826 
16827   ins_pipe(pipe_class_call);
16828 %}
16829 
16830 
16831 // Return Instruction
16832 // epilog node loads ret address into lr as part of frame pop
16833 instruct Ret()
16834 %{
16835   match(Return);
16836 
16837   format %{ "ret\t// return register" %}
16838 
16839   ins_encode( aarch64_enc_ret() );
16840 
16841   ins_pipe(pipe_branch);
16842 %}
16843 
16844 // Die now.
16845 instruct ShouldNotReachHere() %{
16846   match(Halt);
16847 
16848   ins_cost(CALL_COST);
16849   format %{ "ShouldNotReachHere" %}
16850 
16851   ins_encode %{
16852     if (is_reachable()) {
16853       __ stop(_halt_reason);
16854     }
16855   %}
16856 
16857   ins_pipe(pipe_class_default);
16858 %}
16859 
16860 // ============================================================================
16861 // Partial Subtype Check
16862 //
16863 // superklass array for an instance of the superklass.  Set a hidden
16864 // internal cache on a hit (cache is checked with exposed code in
16865 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16866 // encoding ALSO sets flags.
16867 
16868 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16869 %{
16870   match(Set result (PartialSubtypeCheck sub super));
16871   effect(KILL cr, KILL temp);
16872 
16873   ins_cost(1100);  // slightly larger than the next version
16874   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16875 
16876   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16877 
16878   opcode(0x1); // Force zero of result reg on hit
16879 
16880   ins_pipe(pipe_class_memory);
16881 %}
16882 
16883 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16884 %{
16885   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16886   effect(KILL temp, KILL result);
16887 
16888   ins_cost(1100);  // slightly larger than the next version
16889   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16890 
16891   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16892 
16893   opcode(0x0); // Don't zero result reg on hit
16894 
16895   ins_pipe(pipe_class_memory);
16896 %}
16897 
16898 // Intrisics for String.compareTo()
16899 
16900 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16901                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16902 %{
16903   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16904   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16905   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16906 
16907   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16908   ins_encode %{
16909     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16910     __ string_compare($str1$$Register, $str2$$Register,
16911                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16912                       $tmp1$$Register, $tmp2$$Register,
16913                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16914   %}
16915   ins_pipe(pipe_class_memory);
16916 %}
16917 
16918 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16919                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16920 %{
16921   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16922   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16923   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16924 
16925   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16926   ins_encode %{
16927     __ string_compare($str1$$Register, $str2$$Register,
16928                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16929                       $tmp1$$Register, $tmp2$$Register,
16930                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16931   %}
16932   ins_pipe(pipe_class_memory);
16933 %}
16934 
16935 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16936                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16937                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16938 %{
16939   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16940   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16941   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16942          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16943 
16944   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16945   ins_encode %{
16946     __ string_compare($str1$$Register, $str2$$Register,
16947                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16948                       $tmp1$$Register, $tmp2$$Register,
16949                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16950                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16951   %}
16952   ins_pipe(pipe_class_memory);
16953 %}
16954 
16955 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16956                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16957                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16958 %{
16959   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16960   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16961   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16962          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16963 
16964   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16965   ins_encode %{
16966     __ string_compare($str1$$Register, $str2$$Register,
16967                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16968                       $tmp1$$Register, $tmp2$$Register,
16969                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16970                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16971   %}
16972   ins_pipe(pipe_class_memory);
16973 %}
16974 
16975 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16976 // these string_compare variants as NEON register type for convenience so that the prototype of
16977 // string_compare can be shared with all variants.
16978 
16979 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16980                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16981                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16982                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16983 %{
16984   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16985   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16986   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16987          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16988 
16989   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16990   ins_encode %{
16991     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16992     __ string_compare($str1$$Register, $str2$$Register,
16993                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16994                       $tmp1$$Register, $tmp2$$Register,
16995                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16996                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16997                       StrIntrinsicNode::LL);
16998   %}
16999   ins_pipe(pipe_class_memory);
17000 %}
17001 
17002 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17003                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17004                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17005                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17006 %{
17007   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
17008   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17009   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17010          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17011 
17012   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17013   ins_encode %{
17014     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17015     __ string_compare($str1$$Register, $str2$$Register,
17016                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17017                       $tmp1$$Register, $tmp2$$Register,
17018                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17019                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17020                       StrIntrinsicNode::LU);
17021   %}
17022   ins_pipe(pipe_class_memory);
17023 %}
17024 
17025 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17026                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17027                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17028                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17029 %{
17030   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17031   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17032   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17033          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17034 
17035   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17036   ins_encode %{
17037     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17038     __ string_compare($str1$$Register, $str2$$Register,
17039                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17040                       $tmp1$$Register, $tmp2$$Register,
17041                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17042                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17043                       StrIntrinsicNode::UL);
17044   %}
17045   ins_pipe(pipe_class_memory);
17046 %}
17047 
17048 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17049                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17050                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17051                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17052 %{
17053   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17054   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17055   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17056          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17057 
17058   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17059   ins_encode %{
17060     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17061     __ string_compare($str1$$Register, $str2$$Register,
17062                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17063                       $tmp1$$Register, $tmp2$$Register,
17064                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17065                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17066                       StrIntrinsicNode::UU);
17067   %}
17068   ins_pipe(pipe_class_memory);
17069 %}
17070 
17071 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17072                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17073                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17074                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17075 %{
17076   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17077   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17078   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17079          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17080          TEMP vtmp0, TEMP vtmp1, KILL cr);
17081   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17082             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17083 
17084   ins_encode %{
17085     __ string_indexof($str1$$Register, $str2$$Register,
17086                       $cnt1$$Register, $cnt2$$Register,
17087                       $tmp1$$Register, $tmp2$$Register,
17088                       $tmp3$$Register, $tmp4$$Register,
17089                       $tmp5$$Register, $tmp6$$Register,
17090                       -1, $result$$Register, StrIntrinsicNode::UU);
17091   %}
17092   ins_pipe(pipe_class_memory);
17093 %}
17094 
17095 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17096                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17097                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17098                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17099 %{
17100   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17101   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17102   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17103          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17104          TEMP vtmp0, TEMP vtmp1, KILL cr);
17105   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17106             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17107 
17108   ins_encode %{
17109     __ string_indexof($str1$$Register, $str2$$Register,
17110                       $cnt1$$Register, $cnt2$$Register,
17111                       $tmp1$$Register, $tmp2$$Register,
17112                       $tmp3$$Register, $tmp4$$Register,
17113                       $tmp5$$Register, $tmp6$$Register,
17114                       -1, $result$$Register, StrIntrinsicNode::LL);
17115   %}
17116   ins_pipe(pipe_class_memory);
17117 %}
17118 
17119 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17120                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17121                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17122                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17123 %{
17124   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17125   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17126   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17127          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17128          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17129   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17130             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17131 
17132   ins_encode %{
17133     __ string_indexof($str1$$Register, $str2$$Register,
17134                       $cnt1$$Register, $cnt2$$Register,
17135                       $tmp1$$Register, $tmp2$$Register,
17136                       $tmp3$$Register, $tmp4$$Register,
17137                       $tmp5$$Register, $tmp6$$Register,
17138                       -1, $result$$Register, StrIntrinsicNode::UL);
17139   %}
17140   ins_pipe(pipe_class_memory);
17141 %}
17142 
17143 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17144                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17145                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17146 %{
17147   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17148   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17149   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17150          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17151   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17152             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17153 
17154   ins_encode %{
17155     int icnt2 = (int)$int_cnt2$$constant;
17156     __ string_indexof($str1$$Register, $str2$$Register,
17157                       $cnt1$$Register, zr,
17158                       $tmp1$$Register, $tmp2$$Register,
17159                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17160                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17161   %}
17162   ins_pipe(pipe_class_memory);
17163 %}
17164 
17165 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17166                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17167                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17168 %{
17169   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17170   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17171   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17172          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17173   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17174             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17175 
17176   ins_encode %{
17177     int icnt2 = (int)$int_cnt2$$constant;
17178     __ string_indexof($str1$$Register, $str2$$Register,
17179                       $cnt1$$Register, zr,
17180                       $tmp1$$Register, $tmp2$$Register,
17181                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17182                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17183   %}
17184   ins_pipe(pipe_class_memory);
17185 %}
17186 
17187 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17188                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17189                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17190 %{
17191   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17192   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17193   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17194          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17195   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17196             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17197 
17198   ins_encode %{
17199     int icnt2 = (int)$int_cnt2$$constant;
17200     __ string_indexof($str1$$Register, $str2$$Register,
17201                       $cnt1$$Register, zr,
17202                       $tmp1$$Register, $tmp2$$Register,
17203                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17204                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17205   %}
17206   ins_pipe(pipe_class_memory);
17207 %}
17208 
17209 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17210                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17211                              iRegINoSp tmp3, rFlagsReg cr)
17212 %{
17213   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17214   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17215   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17216          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17217 
17218   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17219 
17220   ins_encode %{
17221     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17222                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17223                            $tmp3$$Register);
17224   %}
17225   ins_pipe(pipe_class_memory);
17226 %}
17227 
17228 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17229                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17230                               iRegINoSp tmp3, rFlagsReg cr)
17231 %{
17232   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17233   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17234   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17235          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17236 
17237   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17238 
17239   ins_encode %{
17240     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17241                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17242                             $tmp3$$Register);
17243   %}
17244   ins_pipe(pipe_class_memory);
17245 %}
17246 
17247 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17248                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17249                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17250   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17251   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17252   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17253   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17254   ins_encode %{
17255     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17256                                $result$$Register, $ztmp1$$FloatRegister,
17257                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17258                                $ptmp$$PRegister, true /* isL */);
17259   %}
17260   ins_pipe(pipe_class_memory);
17261 %}
17262 
17263 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17264                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17265                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17266   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17267   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17268   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17269   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17270   ins_encode %{
17271     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17272                                $result$$Register, $ztmp1$$FloatRegister,
17273                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17274                                $ptmp$$PRegister, false /* isL */);
17275   %}
17276   ins_pipe(pipe_class_memory);
17277 %}
17278 
17279 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17280                         iRegI_R0 result, rFlagsReg cr)
17281 %{
17282   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17283   match(Set result (StrEquals (Binary str1 str2) cnt));
17284   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17285 
17286   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17287   ins_encode %{
17288     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17289     __ string_equals($str1$$Register, $str2$$Register,
17290                      $result$$Register, $cnt$$Register, 1);
17291   %}
17292   ins_pipe(pipe_class_memory);
17293 %}
17294 
17295 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17296                         iRegI_R0 result, rFlagsReg cr)
17297 %{
17298   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17299   match(Set result (StrEquals (Binary str1 str2) cnt));
17300   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17301 
17302   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17303   ins_encode %{
17304     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17305     __ string_equals($str1$$Register, $str2$$Register,
17306                      $result$$Register, $cnt$$Register, 2);
17307   %}
17308   ins_pipe(pipe_class_memory);
17309 %}
17310 
17311 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17312                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17313                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17314                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17315                        iRegP_R10 tmp, rFlagsReg cr)
17316 %{
17317   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17318   match(Set result (AryEq ary1 ary2));
17319   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17320          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17321          TEMP vtmp6, TEMP vtmp7, KILL cr);
17322 
17323   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17324   ins_encode %{
17325     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17326                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17327                                    $result$$Register, $tmp$$Register, 1);
17328     if (tpc == NULL) {
17329       ciEnv::current()->record_failure("CodeCache is full");
17330       return;
17331     }
17332   %}
17333   ins_pipe(pipe_class_memory);
17334 %}
17335 
17336 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17337                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17338                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17339                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17340                        iRegP_R10 tmp, rFlagsReg cr)
17341 %{
17342   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17343   match(Set result (AryEq ary1 ary2));
17344   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17345          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17346          TEMP vtmp6, TEMP vtmp7, KILL cr);
17347 
17348   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17349   ins_encode %{
17350     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17351                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17352                                    $result$$Register, $tmp$$Register, 2);
17353     if (tpc == NULL) {
17354       ciEnv::current()->record_failure("CodeCache is full");
17355       return;
17356     }
17357   %}
17358   ins_pipe(pipe_class_memory);
17359 %}
17360 
17361 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17362 %{
17363   match(Set result (CountPositives ary1 len));
17364   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17365   format %{ "count positives byte[] $ary1,$len -> $result" %}
17366   ins_encode %{
17367     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17368     if (tpc == NULL) {
17369       ciEnv::current()->record_failure("CodeCache is full");
17370       return;
17371     }
17372   %}
17373   ins_pipe( pipe_slow );
17374 %}
17375 
17376 // fast char[] to byte[] compression
17377 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17378                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17379                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17380                          iRegI_R0 result, rFlagsReg cr)
17381 %{
17382   match(Set result (StrCompressedCopy src (Binary dst len)));
17383   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17384          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17385 
17386   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17387   ins_encode %{
17388     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17389                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17390                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17391                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17392   %}
17393   ins_pipe(pipe_slow);
17394 %}
17395 
17396 // fast byte[] to char[] inflation
17397 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17398                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17399                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17400 %{
17401   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17402   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17403          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17404          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17405 
17406   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17407   ins_encode %{
17408     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17409                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17410                                         $vtmp2$$FloatRegister, $tmp$$Register);
17411     if (tpc == NULL) {
17412       ciEnv::current()->record_failure("CodeCache is full");
17413       return;
17414     }
17415   %}
17416   ins_pipe(pipe_class_memory);
17417 %}
17418 
17419 // encode char[] to byte[] in ISO_8859_1
17420 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17421                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17422                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17423                           iRegI_R0 result, rFlagsReg cr)
17424 %{
17425   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17426   match(Set result (EncodeISOArray src (Binary dst len)));
17427   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17428          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17429 
17430   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17431   ins_encode %{
17432     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17433                         $result$$Register, false,
17434                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17435                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17436                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17437   %}
17438   ins_pipe(pipe_class_memory);
17439 %}
17440 
17441 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17442                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17443                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17444                             iRegI_R0 result, rFlagsReg cr)
17445 %{
17446   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17447   match(Set result (EncodeISOArray src (Binary dst len)));
17448   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17449          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17450 
17451   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17452   ins_encode %{
17453     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17454                         $result$$Register, true,
17455                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17456                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17457                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17458   %}
17459   ins_pipe(pipe_class_memory);
17460 %}
17461 
17462 //----------------------------- CompressBits/ExpandBits ------------------------
17463 
17464 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17465                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17466   match(Set dst (CompressBits src mask));
17467   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17468   format %{ "mov    $tsrc, $src\n\t"
17469             "mov    $tmask, $mask\n\t"
17470             "bext   $tdst, $tsrc, $tmask\n\t"
17471             "mov    $dst, $tdst"
17472           %}
17473   ins_encode %{
17474     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17475     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17476     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17477     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17478   %}
17479   ins_pipe(pipe_slow);
17480 %}
17481 
17482 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17483                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17484   match(Set dst (CompressBits (LoadI mem) mask));
17485   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17486   format %{ "ldrs   $tsrc, $mem\n\t"
17487             "ldrs   $tmask, $mask\n\t"
17488             "bext   $tdst, $tsrc, $tmask\n\t"
17489             "mov    $dst, $tdst"
17490           %}
17491   ins_encode %{
17492     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17493               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17494     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17495     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17496     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17497   %}
17498   ins_pipe(pipe_slow);
17499 %}
17500 
17501 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17502                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17503   match(Set dst (CompressBits src mask));
17504   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17505   format %{ "mov    $tsrc, $src\n\t"
17506             "mov    $tmask, $mask\n\t"
17507             "bext   $tdst, $tsrc, $tmask\n\t"
17508             "mov    $dst, $tdst"
17509           %}
17510   ins_encode %{
17511     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17512     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17513     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17514     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17515   %}
17516   ins_pipe(pipe_slow);
17517 %}
17518 
17519 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17520                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17521   match(Set dst (CompressBits (LoadL mem) mask));
17522   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17523   format %{ "ldrd   $tsrc, $mem\n\t"
17524             "ldrd   $tmask, $mask\n\t"
17525             "bext   $tdst, $tsrc, $tmask\n\t"
17526             "mov    $dst, $tdst"
17527           %}
17528   ins_encode %{
17529     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17530               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17531     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17532     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17533     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17534   %}
17535   ins_pipe(pipe_slow);
17536 %}
17537 
17538 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17539                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17540   match(Set dst (ExpandBits src mask));
17541   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17542   format %{ "mov    $tsrc, $src\n\t"
17543             "mov    $tmask, $mask\n\t"
17544             "bdep   $tdst, $tsrc, $tmask\n\t"
17545             "mov    $dst, $tdst"
17546           %}
17547   ins_encode %{
17548     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17549     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17550     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17551     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17552   %}
17553   ins_pipe(pipe_slow);
17554 %}
17555 
17556 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17557                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17558   match(Set dst (ExpandBits (LoadI mem) mask));
17559   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17560   format %{ "ldrs   $tsrc, $mem\n\t"
17561             "ldrs   $tmask, $mask\n\t"
17562             "bdep   $tdst, $tsrc, $tmask\n\t"
17563             "mov    $dst, $tdst"
17564           %}
17565   ins_encode %{
17566     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17567               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17568     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17569     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17570     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17571   %}
17572   ins_pipe(pipe_slow);
17573 %}
17574 
17575 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17576                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17577   match(Set dst (ExpandBits src mask));
17578   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17579   format %{ "mov    $tsrc, $src\n\t"
17580             "mov    $tmask, $mask\n\t"
17581             "bdep   $tdst, $tsrc, $tmask\n\t"
17582             "mov    $dst, $tdst"
17583           %}
17584   ins_encode %{
17585     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17586     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17587     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17588     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17589   %}
17590   ins_pipe(pipe_slow);
17591 %}
17592 
17593 
17594 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17595                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17596   match(Set dst (ExpandBits (LoadL mem) mask));
17597   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17598   format %{ "ldrd   $tsrc, $mem\n\t"
17599             "ldrd   $tmask, $mask\n\t"
17600             "bdep   $tdst, $tsrc, $tmask\n\t"
17601             "mov    $dst, $tdst"
17602           %}
17603   ins_encode %{
17604     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17605               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17606     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17607     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17608     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17609   %}
17610   ins_pipe(pipe_slow);
17611 %}
17612 
17613 // ============================================================================
17614 // This name is KNOWN by the ADLC and cannot be changed.
17615 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17616 // for this guy.
17617 instruct tlsLoadP(thread_RegP dst)
17618 %{
17619   match(Set dst (ThreadLocal));
17620 
17621   ins_cost(0);
17622 
17623   format %{ " -- \t// $dst=Thread::current(), empty" %}
17624 
17625   size(0);
17626 
17627   ins_encode( /*empty*/ );
17628 
17629   ins_pipe(pipe_class_empty);
17630 %}
17631 
17632 //----------PEEPHOLE RULES-----------------------------------------------------
17633 // These must follow all instruction definitions as they use the names
17634 // defined in the instructions definitions.
17635 //
17636 // peepmatch ( root_instr_name [preceding_instruction]* );
17637 //
17638 // peepconstraint %{
17639 // (instruction_number.operand_name relational_op instruction_number.operand_name
17640 //  [, ...] );
17641 // // instruction numbers are zero-based using left to right order in peepmatch
17642 //
17643 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17644 // // provide an instruction_number.operand_name for each operand that appears
17645 // // in the replacement instruction's match rule
17646 //
17647 // ---------VM FLAGS---------------------------------------------------------
17648 //
17649 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17650 //
17651 // Each peephole rule is given an identifying number starting with zero and
17652 // increasing by one in the order seen by the parser.  An individual peephole
17653 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17654 // on the command-line.
17655 //
17656 // ---------CURRENT LIMITATIONS----------------------------------------------
17657 //
17658 // Only match adjacent instructions in same basic block
17659 // Only equality constraints
17660 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17661 // Only one replacement instruction
17662 //
17663 // ---------EXAMPLE----------------------------------------------------------
17664 //
17665 // // pertinent parts of existing instructions in architecture description
17666 // instruct movI(iRegINoSp dst, iRegI src)
17667 // %{
17668 //   match(Set dst (CopyI src));
17669 // %}
17670 //
17671 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17672 // %{
17673 //   match(Set dst (AddI dst src));
17674 //   effect(KILL cr);
17675 // %}
17676 //
17677 // // Change (inc mov) to lea
17678 // peephole %{
17679 //   // increment preceded by register-register move
17680 //   peepmatch ( incI_iReg movI );
17681 //   // require that the destination register of the increment
17682 //   // match the destination register of the move
17683 //   peepconstraint ( 0.dst == 1.dst );
17684 //   // construct a replacement instruction that sets
17685 //   // the destination to ( move's source register + one )
17686 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17687 // %}
17688 //
17689 
17690 // Implementation no longer uses movX instructions since
17691 // machine-independent system no longer uses CopyX nodes.
17692 //
17693 // peephole
17694 // %{
17695 //   peepmatch (incI_iReg movI);
17696 //   peepconstraint (0.dst == 1.dst);
17697 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17698 // %}
17699 
17700 // peephole
17701 // %{
17702 //   peepmatch (decI_iReg movI);
17703 //   peepconstraint (0.dst == 1.dst);
17704 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17705 // %}
17706 
17707 // peephole
17708 // %{
17709 //   peepmatch (addI_iReg_imm movI);
17710 //   peepconstraint (0.dst == 1.dst);
17711 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17712 // %}
17713 
17714 // peephole
17715 // %{
17716 //   peepmatch (incL_iReg movL);
17717 //   peepconstraint (0.dst == 1.dst);
17718 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17719 // %}
17720 
17721 // peephole
17722 // %{
17723 //   peepmatch (decL_iReg movL);
17724 //   peepconstraint (0.dst == 1.dst);
17725 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17726 // %}
17727 
17728 // peephole
17729 // %{
17730 //   peepmatch (addL_iReg_imm movL);
17731 //   peepconstraint (0.dst == 1.dst);
17732 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17733 // %}
17734 
17735 // peephole
17736 // %{
17737 //   peepmatch (addP_iReg_imm movP);
17738 //   peepconstraint (0.dst == 1.dst);
17739 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17740 // %}
17741 
17742 // // Change load of spilled value to only a spill
17743 // instruct storeI(memory mem, iRegI src)
17744 // %{
17745 //   match(Set mem (StoreI mem src));
17746 // %}
17747 //
17748 // instruct loadI(iRegINoSp dst, memory mem)
17749 // %{
17750 //   match(Set dst (LoadI mem));
17751 // %}
17752 //
17753 
17754 //----------SMARTSPILL RULES---------------------------------------------------
17755 // These must follow all instruction definitions as they use the names
17756 // defined in the instructions definitions.
17757 
17758 // Local Variables:
17759 // mode: c++
17760 // End: