1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for heapbase register
  668 reg_class heapbase_reg(
  669     R27, R27_H
  670 );
  671 
  672 // Class for thread register
  673 reg_class thread_reg(
  674     R28, R28_H
  675 );
  676 
  677 // Class for frame pointer register
  678 reg_class fp_reg(
  679     R29, R29_H
  680 );
  681 
  682 // Class for link register
  683 reg_class lr_reg(
  684     R30, R30_H
  685 );
  686 
  687 // Class for long sp register
  688 reg_class sp_reg(
  689   R31, R31_H
  690 );
  691 
  692 // Class for all pointer registers
  693 reg_class ptr_reg %{
  694   return _PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers
  698 reg_class no_special_ptr_reg %{
  699   return _NO_SPECIAL_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 
 1134 class CallStubImpl {
 1135 
 1136   //--------------------------------------------------------------
 1137   //---<  Used for optimization in Compile::shorten_branches  >---
 1138   //--------------------------------------------------------------
 1139 
 1140  public:
 1141   // Size of call trampoline stub.
 1142   static uint size_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 
 1146   // number of relocations needed by a call trampoline stub
 1147   static uint reloc_call_trampoline() {
 1148     return 0; // no call trampolines on this platform
 1149   }
 1150 };
 1151 
 1152 class HandlerImpl {
 1153 
 1154  public:
 1155 
 1156   static int emit_exception_handler(CodeBuffer &cbuf);
 1157   static int emit_deopt_handler(CodeBuffer& cbuf);
 1158 
 1159   static uint size_exception_handler() {
 1160     return MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 
 1163   static uint size_deopt_handler() {
 1164     // count one adr and one far branch instruction
 1165     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1166   }
 1167 };
 1168 
 1169 class Node::PD {
 1170 public:
 1171   enum NodeFlags {
 1172     _last_flag = Node::_last_flag
 1173   };
 1174 };
 1175 
 1176   bool is_CAS(int opcode, bool maybe_volatile);
 1177 
 1178   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1179 
 1180   bool unnecessary_acquire(const Node *barrier);
 1181   bool needs_acquiring_load(const Node *load);
 1182 
 1183   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1184 
 1185   bool unnecessary_release(const Node *barrier);
 1186   bool unnecessary_volatile(const Node *barrier);
 1187   bool needs_releasing_store(const Node *store);
 1188 
 1189   // predicate controlling translation of CompareAndSwapX
 1190   bool needs_acquiring_load_exclusive(const Node *load);
 1191 
 1192   // predicate controlling addressing modes
 1193   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1194 %}
 1195 
 1196 source %{
 1197 
 1198   // Derived RegMask with conditionally allocatable registers
 1199 
 1200   void PhaseOutput::pd_perform_mach_node_analysis() {
 1201   }
 1202 
 1203   int MachNode::pd_alignment_required() const {
 1204     return 1;
 1205   }
 1206 
 1207   int MachNode::compute_padding(int current_offset) const {
 1208     return 0;
 1209   }
 1210 
 1211   RegMask _ANY_REG32_mask;
 1212   RegMask _ANY_REG_mask;
 1213   RegMask _PTR_REG_mask;
 1214   RegMask _NO_SPECIAL_REG32_mask;
 1215   RegMask _NO_SPECIAL_REG_mask;
 1216   RegMask _NO_SPECIAL_PTR_REG_mask;
 1217 
 1218   void reg_mask_init() {
 1219     // We derive below RegMask(s) from the ones which are auto-generated from
 1220     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1221     // registers conditionally reserved.
 1222 
 1223     _ANY_REG32_mask = _ALL_REG32_mask;
 1224     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1225 
 1226     _ANY_REG_mask = _ALL_REG_mask;
 1227 
 1228     _PTR_REG_mask = _ALL_REG_mask;
 1229 
 1230     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1231     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1232 
 1233     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1234     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1235 
 1236     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1237     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1238 
 1239     // r27 is not allocatable when compressed oops is on and heapbase is not
 1240     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1241     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1242       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1244       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1245     }
 1246 
 1247     // r29 is not allocatable when PreserveFramePointer is on
 1248     if (PreserveFramePointer) {
 1249       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1251       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1252     }
 1253   }
 1254 
 1255   // Optimizaton of volatile gets and puts
 1256   // -------------------------------------
 1257   //
 1258   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1259   // use to implement volatile reads and writes. For a volatile read
 1260   // we simply need
 1261   //
 1262   //   ldar<x>
 1263   //
 1264   // and for a volatile write we need
 1265   //
 1266   //   stlr<x>
 1267   //
 1268   // Alternatively, we can implement them by pairing a normal
 1269   // load/store with a memory barrier. For a volatile read we need
 1270   //
 1271   //   ldr<x>
 1272   //   dmb ishld
 1273   //
 1274   // for a volatile write
 1275   //
 1276   //   dmb ish
 1277   //   str<x>
 1278   //   dmb ish
 1279   //
 1280   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1281   // sequences. These are normally translated to an instruction
 1282   // sequence like the following
 1283   //
 1284   //   dmb      ish
 1285   // retry:
 1286   //   ldxr<x>   rval raddr
 1287   //   cmp       rval rold
 1288   //   b.ne done
 1289   //   stlxr<x>  rval, rnew, rold
 1290   //   cbnz      rval retry
 1291   // done:
 1292   //   cset      r0, eq
 1293   //   dmb ishld
 1294   //
 1295   // Note that the exclusive store is already using an stlxr
 1296   // instruction. That is required to ensure visibility to other
 1297   // threads of the exclusive write (assuming it succeeds) before that
 1298   // of any subsequent writes.
 1299   //
 1300   // The following instruction sequence is an improvement on the above
 1301   //
 1302   // retry:
 1303   //   ldaxr<x>  rval raddr
 1304   //   cmp       rval rold
 1305   //   b.ne done
 1306   //   stlxr<x>  rval, rnew, rold
 1307   //   cbnz      rval retry
 1308   // done:
 1309   //   cset      r0, eq
 1310   //
 1311   // We don't need the leading dmb ish since the stlxr guarantees
 1312   // visibility of prior writes in the case that the swap is
 1313   // successful. Crucially we don't have to worry about the case where
 1314   // the swap is not successful since no valid program should be
 1315   // relying on visibility of prior changes by the attempting thread
 1316   // in the case where the CAS fails.
 1317   //
 1318   // Similarly, we don't need the trailing dmb ishld if we substitute
 1319   // an ldaxr instruction since that will provide all the guarantees we
 1320   // require regarding observation of changes made by other threads
 1321   // before any change to the CAS address observed by the load.
 1322   //
 1323   // In order to generate the desired instruction sequence we need to
 1324   // be able to identify specific 'signature' ideal graph node
 1325   // sequences which i) occur as a translation of a volatile reads or
 1326   // writes or CAS operations and ii) do not occur through any other
 1327   // translation or graph transformation. We can then provide
 1328   // alternative aldc matching rules which translate these node
 1329   // sequences to the desired machine code sequences. Selection of the
 1330   // alternative rules can be implemented by predicates which identify
 1331   // the relevant node sequences.
 1332   //
 1333   // The ideal graph generator translates a volatile read to the node
 1334   // sequence
 1335   //
 1336   //   LoadX[mo_acquire]
 1337   //   MemBarAcquire
 1338   //
 1339   // As a special case when using the compressed oops optimization we
 1340   // may also see this variant
 1341   //
 1342   //   LoadN[mo_acquire]
 1343   //   DecodeN
 1344   //   MemBarAcquire
 1345   //
 1346   // A volatile write is translated to the node sequence
 1347   //
 1348   //   MemBarRelease
 1349   //   StoreX[mo_release] {CardMark}-optional
 1350   //   MemBarVolatile
 1351   //
 1352   // n.b. the above node patterns are generated with a strict
 1353   // 'signature' configuration of input and output dependencies (see
 1354   // the predicates below for exact details). The card mark may be as
 1355   // simple as a few extra nodes or, in a few GC configurations, may
 1356   // include more complex control flow between the leading and
 1357   // trailing memory barriers. However, whatever the card mark
 1358   // configuration these signatures are unique to translated volatile
 1359   // reads/stores -- they will not appear as a result of any other
 1360   // bytecode translation or inlining nor as a consequence of
 1361   // optimizing transforms.
 1362   //
 1363   // We also want to catch inlined unsafe volatile gets and puts and
 1364   // be able to implement them using either ldar<x>/stlr<x> or some
 1365   // combination of ldr<x>/stlr<x> and dmb instructions.
 1366   //
 1367   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1368   // normal volatile put node sequence containing an extra cpuorder
 1369   // membar
 1370   //
 1371   //   MemBarRelease
 1372   //   MemBarCPUOrder
 1373   //   StoreX[mo_release] {CardMark}-optional
 1374   //   MemBarCPUOrder
 1375   //   MemBarVolatile
 1376   //
 1377   // n.b. as an aside, a cpuorder membar is not itself subject to
 1378   // matching and translation by adlc rules.  However, the rule
 1379   // predicates need to detect its presence in order to correctly
 1380   // select the desired adlc rules.
 1381   //
 1382   // Inlined unsafe volatile gets manifest as a slightly different
 1383   // node sequence to a normal volatile get because of the
 1384   // introduction of some CPUOrder memory barriers to bracket the
 1385   // Load. However, but the same basic skeleton of a LoadX feeding a
 1386   // MemBarAcquire, possibly through an optional DecodeN, is still
 1387   // present
 1388   //
 1389   //   MemBarCPUOrder
 1390   //        ||       \\
 1391   //   MemBarCPUOrder LoadX[mo_acquire]
 1392   //        ||            |
 1393   //        ||       {DecodeN} optional
 1394   //        ||       /
 1395   //     MemBarAcquire
 1396   //
 1397   // In this case the acquire membar does not directly depend on the
 1398   // load. However, we can be sure that the load is generated from an
 1399   // inlined unsafe volatile get if we see it dependent on this unique
 1400   // sequence of membar nodes. Similarly, given an acquire membar we
 1401   // can know that it was added because of an inlined unsafe volatile
 1402   // get if it is fed and feeds a cpuorder membar and if its feed
 1403   // membar also feeds an acquiring load.
 1404   //
 1405   // Finally an inlined (Unsafe) CAS operation is translated to the
 1406   // following ideal graph
 1407   //
 1408   //   MemBarRelease
 1409   //   MemBarCPUOrder
 1410   //   CompareAndSwapX {CardMark}-optional
 1411   //   MemBarCPUOrder
 1412   //   MemBarAcquire
 1413   //
 1414   // So, where we can identify these volatile read and write
 1415   // signatures we can choose to plant either of the above two code
 1416   // sequences. For a volatile read we can simply plant a normal
 1417   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1418   // also choose to inhibit translation of the MemBarAcquire and
 1419   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1420   //
 1421   // When we recognise a volatile store signature we can choose to
 1422   // plant at a dmb ish as a translation for the MemBarRelease, a
 1423   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1424   // Alternatively, we can inhibit translation of the MemBarRelease
 1425   // and MemBarVolatile and instead plant a simple stlr<x>
 1426   // instruction.
 1427   //
 1428   // when we recognise a CAS signature we can choose to plant a dmb
 1429   // ish as a translation for the MemBarRelease, the conventional
 1430   // macro-instruction sequence for the CompareAndSwap node (which
 1431   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1432   // Alternatively, we can elide generation of the dmb instructions
 1433   // and plant the alternative CompareAndSwap macro-instruction
 1434   // sequence (which uses ldaxr<x>).
 1435   //
 1436   // Of course, the above only applies when we see these signature
 1437   // configurations. We still want to plant dmb instructions in any
 1438   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1439   // MemBarVolatile. For example, at the end of a constructor which
 1440   // writes final/volatile fields we will see a MemBarRelease
 1441   // instruction and this needs a 'dmb ish' lest we risk the
 1442   // constructed object being visible without making the
 1443   // final/volatile field writes visible.
 1444   //
 1445   // n.b. the translation rules below which rely on detection of the
 1446   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1447   // If we see anything other than the signature configurations we
 1448   // always just translate the loads and stores to ldr<x> and str<x>
 1449   // and translate acquire, release and volatile membars to the
 1450   // relevant dmb instructions.
 1451   //
 1452 
 1453   // is_CAS(int opcode, bool maybe_volatile)
 1454   //
 1455   // return true if opcode is one of the possible CompareAndSwapX
 1456   // values otherwise false.
 1457 
 1458   bool is_CAS(int opcode, bool maybe_volatile)
 1459   {
 1460     switch(opcode) {
 1461       // We handle these
 1462     case Op_CompareAndSwapI:
 1463     case Op_CompareAndSwapL:
 1464     case Op_CompareAndSwapP:
 1465     case Op_CompareAndSwapN:
 1466     case Op_ShenandoahCompareAndSwapP:
 1467     case Op_ShenandoahCompareAndSwapN:
 1468     case Op_CompareAndSwapB:
 1469     case Op_CompareAndSwapS:
 1470     case Op_GetAndSetI:
 1471     case Op_GetAndSetL:
 1472     case Op_GetAndSetP:
 1473     case Op_GetAndSetN:
 1474     case Op_GetAndAddI:
 1475     case Op_GetAndAddL:
 1476       return true;
 1477     case Op_CompareAndExchangeI:
 1478     case Op_CompareAndExchangeN:
 1479     case Op_CompareAndExchangeB:
 1480     case Op_CompareAndExchangeS:
 1481     case Op_CompareAndExchangeL:
 1482     case Op_CompareAndExchangeP:
 1483     case Op_WeakCompareAndSwapB:
 1484     case Op_WeakCompareAndSwapS:
 1485     case Op_WeakCompareAndSwapI:
 1486     case Op_WeakCompareAndSwapL:
 1487     case Op_WeakCompareAndSwapP:
 1488     case Op_WeakCompareAndSwapN:
 1489     case Op_ShenandoahWeakCompareAndSwapP:
 1490     case Op_ShenandoahWeakCompareAndSwapN:
 1491     case Op_ShenandoahCompareAndExchangeP:
 1492     case Op_ShenandoahCompareAndExchangeN:
 1493       return maybe_volatile;
 1494     default:
 1495       return false;
 1496     }
 1497   }
 1498 
 1499   // helper to determine the maximum number of Phi nodes we may need to
 1500   // traverse when searching from a card mark membar for the merge mem
 1501   // feeding a trailing membar or vice versa
 1502 
 1503 // predicates controlling emit of ldr<x>/ldar<x>
 1504 
 1505 bool unnecessary_acquire(const Node *barrier)
 1506 {
 1507   assert(barrier->is_MemBar(), "expecting a membar");
 1508 
 1509   MemBarNode* mb = barrier->as_MemBar();
 1510 
 1511   if (mb->trailing_load()) {
 1512     return true;
 1513   }
 1514 
 1515   if (mb->trailing_load_store()) {
 1516     Node* load_store = mb->in(MemBarNode::Precedent);
 1517     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1518     return is_CAS(load_store->Opcode(), true);
 1519   }
 1520 
 1521   return false;
 1522 }
 1523 
 1524 bool needs_acquiring_load(const Node *n)
 1525 {
 1526   assert(n->is_Load(), "expecting a load");
 1527   LoadNode *ld = n->as_Load();
 1528   return ld->is_acquire();
 1529 }
 1530 
 1531 bool unnecessary_release(const Node *n)
 1532 {
 1533   assert((n->is_MemBar() &&
 1534           n->Opcode() == Op_MemBarRelease),
 1535          "expecting a release membar");
 1536 
 1537   MemBarNode *barrier = n->as_MemBar();
 1538   if (!barrier->leading()) {
 1539     return false;
 1540   } else {
 1541     Node* trailing = barrier->trailing_membar();
 1542     MemBarNode* trailing_mb = trailing->as_MemBar();
 1543     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1544     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1545 
 1546     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1547     if (mem->is_Store()) {
 1548       assert(mem->as_Store()->is_release(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1550       return true;
 1551     } else {
 1552       assert(mem->is_LoadStore(), "");
 1553       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1554       return is_CAS(mem->Opcode(), true);
 1555     }
 1556   }
 1557   return false;
 1558 }
 1559 
 1560 bool unnecessary_volatile(const Node *n)
 1561 {
 1562   // assert n->is_MemBar();
 1563   MemBarNode *mbvol = n->as_MemBar();
 1564 
 1565   bool release = mbvol->trailing_store();
 1566   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1567 #ifdef ASSERT
 1568   if (release) {
 1569     Node* leading = mbvol->leading_membar();
 1570     assert(leading->Opcode() == Op_MemBarRelease, "");
 1571     assert(leading->as_MemBar()->leading_store(), "");
 1572     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1573   }
 1574 #endif
 1575 
 1576   return release;
 1577 }
 1578 
 1579 // predicates controlling emit of str<x>/stlr<x>
 1580 
 1581 bool needs_releasing_store(const Node *n)
 1582 {
 1583   // assert n->is_Store();
 1584   StoreNode *st = n->as_Store();
 1585   return st->trailing_membar() != NULL;
 1586 }
 1587 
 1588 // predicate controlling translation of CAS
 1589 //
 1590 // returns true if CAS needs to use an acquiring load otherwise false
 1591 
 1592 bool needs_acquiring_load_exclusive(const Node *n)
 1593 {
 1594   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1595   LoadStoreNode* ldst = n->as_LoadStore();
 1596   if (is_CAS(n->Opcode(), false)) {
 1597     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1598   } else {
 1599     return ldst->trailing_membar() != NULL;
 1600   }
 1601 
 1602   // so we can just return true here
 1603   return true;
 1604 }
 1605 
 1606 #define __ _masm.
 1607 
 1608 // advance declarations for helper functions to convert register
 1609 // indices to register objects
 1610 
 1611 // the ad file has to provide implementations of certain methods
 1612 // expected by the generic code
 1613 //
 1614 // REQUIRED FUNCTIONALITY
 1615 
 1616 //=============================================================================
 1617 
 1618 // !!!!! Special hack to get all types of calls to specify the byte offset
 1619 //       from the start of the call to the point where the return address
 1620 //       will point.
 1621 
 1622 int MachCallStaticJavaNode::ret_addr_offset()
 1623 {
 1624   // call should be a simple bl
 1625   int off = 4;
 1626   return off;
 1627 }
 1628 
 1629 int MachCallDynamicJavaNode::ret_addr_offset()
 1630 {
 1631   return 16; // movz, movk, movk, bl
 1632 }
 1633 
 1634 int MachCallRuntimeNode::ret_addr_offset() {
 1635   // for generated stubs the call will be
 1636   //   bl(addr)
 1637   // or with far branches
 1638   //   bl(trampoline_stub)
 1639   // for real runtime callouts it will be six instructions
 1640   // see aarch64_enc_java_to_runtime
 1641   //   adr(rscratch2, retaddr)
 1642   //   lea(rscratch1, RuntimeAddress(addr)
 1643   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1644   //   blr(rscratch1)
 1645   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1646   if (cb) {
 1647     return 1 * NativeInstruction::instruction_size;
 1648   } else {
 1649     return 6 * NativeInstruction::instruction_size;
 1650   }
 1651 }
 1652 
 1653 //=============================================================================
 1654 
 1655 #ifndef PRODUCT
 1656 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1657   st->print("BREAKPOINT");
 1658 }
 1659 #endif
 1660 
 1661 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1662   C2_MacroAssembler _masm(&cbuf);
 1663   __ brk(0);
 1664 }
 1665 
 1666 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1667   return MachNode::size(ra_);
 1668 }
 1669 
 1670 //=============================================================================
 1671 
 1672 #ifndef PRODUCT
 1673   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1674     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1675   }
 1676 #endif
 1677 
 1678   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1679     C2_MacroAssembler _masm(&cbuf);
 1680     for (int i = 0; i < _count; i++) {
 1681       __ nop();
 1682     }
 1683   }
 1684 
 1685   uint MachNopNode::size(PhaseRegAlloc*) const {
 1686     return _count * NativeInstruction::instruction_size;
 1687   }
 1688 
 1689 //=============================================================================
 1690 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1691 
 1692 int ConstantTable::calculate_table_base_offset() const {
 1693   return 0;  // absolute addressing, no offset
 1694 }
 1695 
 1696 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1697 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1698   ShouldNotReachHere();
 1699 }
 1700 
 1701 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1702   // Empty encoding
 1703 }
 1704 
 1705 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1706   return 0;
 1707 }
 1708 
 1709 #ifndef PRODUCT
 1710 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1711   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1712 }
 1713 #endif
 1714 
 1715 #ifndef PRODUCT
 1716 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1717   Compile* C = ra_->C;
 1718 
 1719   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1720 
 1721   if (C->output()->need_stack_bang(framesize))
 1722     st->print("# stack bang size=%d\n\t", framesize);
 1723 
 1724   if (VM_Version::use_rop_protection()) {
 1725     st->print("ldr zr, [lr]\n\t");
 1726     st->print("pacia  lr, rfp\n\t");
 1727   }
 1728   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1729     st->print("sub  sp, sp, #%d\n\t", framesize);
 1730     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1731     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1732   } else {
 1733     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1734     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1735     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1736     st->print("sub  sp, sp, rscratch1");
 1737   }
 1738   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1739     st->print("\n\t");
 1740     st->print("ldr  rscratch1, [guard]\n\t");
 1741     st->print("dmb ishld\n\t");
 1742     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1743     st->print("cmp  rscratch1, rscratch2\n\t");
 1744     st->print("b.eq skip");
 1745     st->print("\n\t");
 1746     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1747     st->print("b skip\n\t");
 1748     st->print("guard: int\n\t");
 1749     st->print("\n\t");
 1750     st->print("skip:\n\t");
 1751   }
 1752 }
 1753 #endif
 1754 
 1755 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1756   Compile* C = ra_->C;
 1757   C2_MacroAssembler _masm(&cbuf);
 1758 
 1759   // n.b. frame size includes space for return pc and rfp
 1760   const int framesize = C->output()->frame_size_in_bytes();
 1761 
 1762   // insert a nop at the start of the prolog so we can patch in a
 1763   // branch if we need to invalidate the method later
 1764   __ nop();
 1765 
 1766   if (C->clinit_barrier_on_entry()) {
 1767     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1768 
 1769     Label L_skip_barrier;
 1770 
 1771     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1772     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1773     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1774     __ bind(L_skip_barrier);
 1775   }
 1776 
 1777   if (C->max_vector_size() > 0) {
 1778     __ reinitialize_ptrue();
 1779   }
 1780 
 1781   int bangsize = C->output()->bang_size_in_bytes();
 1782   if (C->output()->need_stack_bang(bangsize))
 1783     __ generate_stack_overflow_check(bangsize);
 1784 
 1785   __ build_frame(framesize);
 1786 
 1787   int max_monitors = C->method() != NULL ? C->max_monitors() : 0;
 1788   if (UseFastLocking && max_monitors > 0) {
 1789     C2CheckLockStackStub* stub = new (C->comp_arena()) C2CheckLockStackStub();
 1790     C->output()->add_stub(stub);
 1791     __ ldr(r9, Address(rthread, JavaThread::lock_stack_current_offset()));
 1792     __ ldr(r10, Address(rthread, JavaThread::lock_stack_limit_offset()));
 1793     __ add(r9, r9, max_monitors * oopSize);
 1794     __ cmp(r9, r10);
 1795     __ br(Assembler::GE, stub->entry());
 1796     __ bind(stub->continuation());
 1797   }
 1798 
 1799   if (C->stub_function() == NULL) {
 1800     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1801     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1802       // Dummy labels for just measuring the code size
 1803       Label dummy_slow_path;
 1804       Label dummy_continuation;
 1805       Label dummy_guard;
 1806       Label* slow_path = &dummy_slow_path;
 1807       Label* continuation = &dummy_continuation;
 1808       Label* guard = &dummy_guard;
 1809       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1810         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1811         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1812         Compile::current()->output()->add_stub(stub);
 1813         slow_path = &stub->entry();
 1814         continuation = &stub->continuation();
 1815         guard = &stub->guard();
 1816       }
 1817       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1818       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1819     }
 1820   }
 1821 
 1822   if (VerifyStackAtCalls) {
 1823     Unimplemented();
 1824   }
 1825 
 1826   C->output()->set_frame_complete(cbuf.insts_size());
 1827 
 1828   if (C->has_mach_constant_base_node()) {
 1829     // NOTE: We set the table base offset here because users might be
 1830     // emitted before MachConstantBaseNode.
 1831     ConstantTable& constant_table = C->output()->constant_table();
 1832     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1833   }
 1834 }
 1835 
 1836 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1837 {
 1838   return MachNode::size(ra_); // too many variables; just compute it
 1839                               // the hard way
 1840 }
 1841 
 1842 int MachPrologNode::reloc() const
 1843 {
 1844   return 0;
 1845 }
 1846 
 1847 //=============================================================================
 1848 
 1849 #ifndef PRODUCT
 1850 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1851   Compile* C = ra_->C;
 1852   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1853 
 1854   st->print("# pop frame %d\n\t",framesize);
 1855 
 1856   if (framesize == 0) {
 1857     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1858   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1859     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1860     st->print("add  sp, sp, #%d\n\t", framesize);
 1861   } else {
 1862     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1863     st->print("add  sp, sp, rscratch1\n\t");
 1864     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1865   }
 1866   if (VM_Version::use_rop_protection()) {
 1867     st->print("autia lr, rfp\n\t");
 1868     st->print("ldr zr, [lr]\n\t");
 1869   }
 1870 
 1871   if (do_polling() && C->is_method_compilation()) {
 1872     st->print("# test polling word\n\t");
 1873     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1874     st->print("cmp  sp, rscratch1\n\t");
 1875     st->print("bhi #slow_path");
 1876   }
 1877 }
 1878 #endif
 1879 
 1880 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1881   Compile* C = ra_->C;
 1882   C2_MacroAssembler _masm(&cbuf);
 1883   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1884 
 1885   __ remove_frame(framesize);
 1886 
 1887   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1888     __ reserved_stack_check();
 1889   }
 1890 
 1891   if (do_polling() && C->is_method_compilation()) {
 1892     Label dummy_label;
 1893     Label* code_stub = &dummy_label;
 1894     if (!C->output()->in_scratch_emit_size()) {
 1895       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1896       C->output()->add_stub(stub);
 1897       code_stub = &stub->entry();
 1898     }
 1899     __ relocate(relocInfo::poll_return_type);
 1900     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1901   }
 1902 }
 1903 
 1904 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1905   // Variable size. Determine dynamically.
 1906   return MachNode::size(ra_);
 1907 }
 1908 
 1909 int MachEpilogNode::reloc() const {
 1910   // Return number of relocatable values contained in this instruction.
 1911   return 1; // 1 for polling page.
 1912 }
 1913 
 1914 const Pipeline * MachEpilogNode::pipeline() const {
 1915   return MachNode::pipeline_class();
 1916 }
 1917 
 1918 //=============================================================================
 1919 
 1920 // Figure out which register class each belongs in: rc_int, rc_float or
 1921 // rc_stack.
 1922 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1923 
 1924 static enum RC rc_class(OptoReg::Name reg) {
 1925 
 1926   if (reg == OptoReg::Bad) {
 1927     return rc_bad;
 1928   }
 1929 
 1930   // we have 32 int registers * 2 halves
 1931   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1932 
 1933   if (reg < slots_of_int_registers) {
 1934     return rc_int;
 1935   }
 1936 
 1937   // we have 32 float register * 8 halves
 1938   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1939   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1940     return rc_float;
 1941   }
 1942 
 1943   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1944   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1945     return rc_predicate;
 1946   }
 1947 
 1948   // Between predicate regs & stack is the flags.
 1949   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1950 
 1951   return rc_stack;
 1952 }
 1953 
 1954 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1955   Compile* C = ra_->C;
 1956 
 1957   // Get registers to move.
 1958   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1959   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1960   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1961   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1962 
 1963   enum RC src_hi_rc = rc_class(src_hi);
 1964   enum RC src_lo_rc = rc_class(src_lo);
 1965   enum RC dst_hi_rc = rc_class(dst_hi);
 1966   enum RC dst_lo_rc = rc_class(dst_lo);
 1967 
 1968   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1969 
 1970   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1971     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1972            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1973            "expected aligned-adjacent pairs");
 1974   }
 1975 
 1976   if (src_lo == dst_lo && src_hi == dst_hi) {
 1977     return 0;            // Self copy, no move.
 1978   }
 1979 
 1980   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1981               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1982   int src_offset = ra_->reg2offset(src_lo);
 1983   int dst_offset = ra_->reg2offset(dst_lo);
 1984 
 1985   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1986     uint ireg = ideal_reg();
 1987     if (ireg == Op_VecA && cbuf) {
 1988       C2_MacroAssembler _masm(cbuf);
 1989       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1990       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1991         // stack->stack
 1992         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1993                                                 sve_vector_reg_size_in_bytes);
 1994       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1995         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1996                             sve_vector_reg_size_in_bytes);
 1997       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1998         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1999                               sve_vector_reg_size_in_bytes);
 2000       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2001         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2002                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 2003                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 2004       } else {
 2005         ShouldNotReachHere();
 2006       }
 2007     } else if (cbuf) {
 2008       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 2009       C2_MacroAssembler _masm(cbuf);
 2010       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 2011       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2012         // stack->stack
 2013         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2014         if (ireg == Op_VecD) {
 2015           __ unspill(rscratch1, true, src_offset);
 2016           __ spill(rscratch1, true, dst_offset);
 2017         } else {
 2018           __ spill_copy128(src_offset, dst_offset);
 2019         }
 2020       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2021         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2022                ireg == Op_VecD ? __ T8B : __ T16B,
 2023                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2024       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2025         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2026                  ireg == Op_VecD ? __ D : __ Q,
 2027                  ra_->reg2offset(dst_lo));
 2028       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2029         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2030                    ireg == Op_VecD ? __ D : __ Q,
 2031                    ra_->reg2offset(src_lo));
 2032       } else {
 2033         ShouldNotReachHere();
 2034       }
 2035     }
 2036   } else if (cbuf) {
 2037     C2_MacroAssembler _masm(cbuf);
 2038     switch (src_lo_rc) {
 2039     case rc_int:
 2040       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2041         if (is64) {
 2042             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2043                    as_Register(Matcher::_regEncode[src_lo]));
 2044         } else {
 2045             C2_MacroAssembler _masm(cbuf);
 2046             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2047                     as_Register(Matcher::_regEncode[src_lo]));
 2048         }
 2049       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2050         if (is64) {
 2051             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2052                      as_Register(Matcher::_regEncode[src_lo]));
 2053         } else {
 2054             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2055                      as_Register(Matcher::_regEncode[src_lo]));
 2056         }
 2057       } else {                    // gpr --> stack spill
 2058         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2059         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2060       }
 2061       break;
 2062     case rc_float:
 2063       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2064         if (is64) {
 2065             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2066                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2067         } else {
 2068             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2069                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2070         }
 2071       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2072         if (is64) {
 2073             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2074                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2075         } else {
 2076             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2077                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2078         }
 2079       } else {                    // fpr --> stack spill
 2080         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2081         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2082                  is64 ? __ D : __ S, dst_offset);
 2083       }
 2084       break;
 2085     case rc_stack:
 2086       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2087         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2088       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2089         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2090                    is64 ? __ D : __ S, src_offset);
 2091       } else if (dst_lo_rc == rc_predicate) {
 2092         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2093                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2094       } else {                    // stack --> stack copy
 2095         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2096         if (ideal_reg() == Op_RegVectMask) {
 2097           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2098                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2099         } else {
 2100           __ unspill(rscratch1, is64, src_offset);
 2101           __ spill(rscratch1, is64, dst_offset);
 2102         }
 2103       }
 2104       break;
 2105     case rc_predicate:
 2106       if (dst_lo_rc == rc_predicate) {
 2107         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2108       } else if (dst_lo_rc == rc_stack) {
 2109         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2110                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2111       } else {
 2112         assert(false, "bad src and dst rc_class combination.");
 2113         ShouldNotReachHere();
 2114       }
 2115       break;
 2116     default:
 2117       assert(false, "bad rc_class for spill");
 2118       ShouldNotReachHere();
 2119     }
 2120   }
 2121 
 2122   if (st) {
 2123     st->print("spill ");
 2124     if (src_lo_rc == rc_stack) {
 2125       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2126     } else {
 2127       st->print("%s -> ", Matcher::regName[src_lo]);
 2128     }
 2129     if (dst_lo_rc == rc_stack) {
 2130       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2131     } else {
 2132       st->print("%s", Matcher::regName[dst_lo]);
 2133     }
 2134     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2135       int vsize = 0;
 2136       switch (ideal_reg()) {
 2137       case Op_VecD:
 2138         vsize = 64;
 2139         break;
 2140       case Op_VecX:
 2141         vsize = 128;
 2142         break;
 2143       case Op_VecA:
 2144         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2145         break;
 2146       default:
 2147         assert(false, "bad register type for spill");
 2148         ShouldNotReachHere();
 2149       }
 2150       st->print("\t# vector spill size = %d", vsize);
 2151     } else if (ideal_reg() == Op_RegVectMask) {
 2152       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2153       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2154       st->print("\t# predicate spill size = %d", vsize);
 2155     } else {
 2156       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2157     }
 2158   }
 2159 
 2160   return 0;
 2161 
 2162 }
 2163 
 2164 #ifndef PRODUCT
 2165 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2166   if (!ra_)
 2167     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2168   else
 2169     implementation(NULL, ra_, false, st);
 2170 }
 2171 #endif
 2172 
 2173 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2174   implementation(&cbuf, ra_, false, NULL);
 2175 }
 2176 
 2177 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2178   return MachNode::size(ra_);
 2179 }
 2180 
 2181 //=============================================================================
 2182 
 2183 #ifndef PRODUCT
 2184 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2185   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2186   int reg = ra_->get_reg_first(this);
 2187   st->print("add %s, rsp, #%d]\t# box lock",
 2188             Matcher::regName[reg], offset);
 2189 }
 2190 #endif
 2191 
 2192 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2193   C2_MacroAssembler _masm(&cbuf);
 2194 
 2195   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2196   int reg    = ra_->get_encode(this);
 2197 
 2198   // This add will handle any 24-bit signed offset. 24 bits allows an
 2199   // 8 megabyte stack frame.
 2200   __ add(as_Register(reg), sp, offset);
 2201 }
 2202 
 2203 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2204   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2205   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2206 
 2207   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2208     return NativeInstruction::instruction_size;
 2209   } else {
 2210     return 2 * NativeInstruction::instruction_size;
 2211   }
 2212 }
 2213 
 2214 //=============================================================================
 2215 
 2216 #ifndef PRODUCT
 2217 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2218 {
 2219   st->print_cr("# MachUEPNode");
 2220   if (UseCompressedClassPointers) {
 2221     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2222     if (CompressedKlassPointers::shift() != 0) {
 2223       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2224     }
 2225   } else {
 2226    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2227   }
 2228   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2229   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2230 }
 2231 #endif
 2232 
 2233 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2234 {
 2235   // This is the unverified entry point.
 2236   C2_MacroAssembler _masm(&cbuf);
 2237 
 2238   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2239   Label skip;
 2240   // TODO
 2241   // can we avoid this skip and still use a reloc?
 2242   __ br(Assembler::EQ, skip);
 2243   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2244   __ bind(skip);
 2245 }
 2246 
 2247 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2248 {
 2249   return MachNode::size(ra_);
 2250 }
 2251 
 2252 // REQUIRED EMIT CODE
 2253 
 2254 //=============================================================================
 2255 
 2256 // Emit exception handler code.
 2257 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2258 {
 2259   // mov rscratch1 #exception_blob_entry_point
 2260   // br rscratch1
 2261   // Note that the code buffer's insts_mark is always relative to insts.
 2262   // That's why we must use the macroassembler to generate a handler.
 2263   C2_MacroAssembler _masm(&cbuf);
 2264   address base = __ start_a_stub(size_exception_handler());
 2265   if (base == NULL) {
 2266     ciEnv::current()->record_failure("CodeCache is full");
 2267     return 0;  // CodeBuffer::expand failed
 2268   }
 2269   int offset = __ offset();
 2270   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2271   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2272   __ end_a_stub();
 2273   return offset;
 2274 }
 2275 
 2276 // Emit deopt handler code.
 2277 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2278 {
 2279   // Note that the code buffer's insts_mark is always relative to insts.
 2280   // That's why we must use the macroassembler to generate a handler.
 2281   C2_MacroAssembler _masm(&cbuf);
 2282   address base = __ start_a_stub(size_deopt_handler());
 2283   if (base == NULL) {
 2284     ciEnv::current()->record_failure("CodeCache is full");
 2285     return 0;  // CodeBuffer::expand failed
 2286   }
 2287   int offset = __ offset();
 2288 
 2289   __ adr(lr, __ pc());
 2290   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2291 
 2292   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2293   __ end_a_stub();
 2294   return offset;
 2295 }
 2296 
 2297 // REQUIRED MATCHER CODE
 2298 
 2299 //=============================================================================
 2300 
 2301 const bool Matcher::match_rule_supported(int opcode) {
 2302   if (!has_match_rule(opcode))
 2303     return false;
 2304 
 2305   bool ret_value = true;
 2306   switch (opcode) {
 2307     case Op_OnSpinWait:
 2308       return VM_Version::supports_on_spin_wait();
 2309     case Op_CacheWB:
 2310     case Op_CacheWBPreSync:
 2311     case Op_CacheWBPostSync:
 2312       if (!VM_Version::supports_data_cache_line_flush()) {
 2313         ret_value = false;
 2314       }
 2315       break;
 2316   }
 2317 
 2318   return ret_value; // Per default match rules are supported.
 2319 }
 2320 
 2321 const RegMask* Matcher::predicate_reg_mask(void) {
 2322   return &_PR_REG_mask;
 2323 }
 2324 
 2325 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2326   return new TypeVectMask(elemTy, length);
 2327 }
 2328 
 2329 // Vector calling convention not yet implemented.
 2330 const bool Matcher::supports_vector_calling_convention(void) {
 2331   return false;
 2332 }
 2333 
 2334 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2335   Unimplemented();
 2336   return OptoRegPair(0, 0);
 2337 }
 2338 
 2339 // Is this branch offset short enough that a short branch can be used?
 2340 //
 2341 // NOTE: If the platform does not provide any short branch variants, then
 2342 //       this method should return false for offset 0.
 2343 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2344   // The passed offset is relative to address of the branch.
 2345 
 2346   return (-32768 <= offset && offset < 32768);
 2347 }
 2348 
 2349 // Vector width in bytes.
 2350 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2351   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2352   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2353   // Minimum 2 values in vector
 2354   if (size < 2*type2aelembytes(bt)) size = 0;
 2355   // But never < 4
 2356   if (size < 4) size = 0;
 2357   return size;
 2358 }
 2359 
 2360 // Limits on vector size (number of elements) loaded into vector.
 2361 const int Matcher::max_vector_size(const BasicType bt) {
 2362   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2363 }
 2364 
 2365 const int Matcher::min_vector_size(const BasicType bt) {
 2366   int max_size = max_vector_size(bt);
 2367   // Limit the min vector size to 8 bytes.
 2368   int size = 8 / type2aelembytes(bt);
 2369   if (bt == T_BYTE) {
 2370     // To support vector api shuffle/rearrange.
 2371     size = 4;
 2372   } else if (bt == T_BOOLEAN) {
 2373     // To support vector api load/store mask.
 2374     size = 2;
 2375   }
 2376   if (size < 2) size = 2;
 2377   return MIN2(size, max_size);
 2378 }
 2379 
 2380 // Actual max scalable vector register length.
 2381 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2382   return Matcher::max_vector_size(bt);
 2383 }
 2384 
 2385 // Vector ideal reg.
 2386 const uint Matcher::vector_ideal_reg(int len) {
 2387   if (UseSVE > 0 && 16 < len && len <= 256) {
 2388     return Op_VecA;
 2389   }
 2390   switch(len) {
 2391     // For 16-bit/32-bit mask vector, reuse VecD.
 2392     case  2:
 2393     case  4:
 2394     case  8: return Op_VecD;
 2395     case 16: return Op_VecX;
 2396   }
 2397   ShouldNotReachHere();
 2398   return 0;
 2399 }
 2400 
 2401 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2402   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2403   switch (ideal_reg) {
 2404     case Op_VecA: return new vecAOper();
 2405     case Op_VecD: return new vecDOper();
 2406     case Op_VecX: return new vecXOper();
 2407   }
 2408   ShouldNotReachHere();
 2409   return NULL;
 2410 }
 2411 
 2412 bool Matcher::is_reg2reg_move(MachNode* m) {
 2413   return false;
 2414 }
 2415 
 2416 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2417   return opnd->opcode() == VREG;
 2418 }
 2419 
 2420 // Return whether or not this register is ever used as an argument.
 2421 // This function is used on startup to build the trampoline stubs in
 2422 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2423 // call in the trampoline, and arguments in those registers not be
 2424 // available to the callee.
 2425 bool Matcher::can_be_java_arg(int reg)
 2426 {
 2427   return
 2428     reg ==  R0_num || reg == R0_H_num ||
 2429     reg ==  R1_num || reg == R1_H_num ||
 2430     reg ==  R2_num || reg == R2_H_num ||
 2431     reg ==  R3_num || reg == R3_H_num ||
 2432     reg ==  R4_num || reg == R4_H_num ||
 2433     reg ==  R5_num || reg == R5_H_num ||
 2434     reg ==  R6_num || reg == R6_H_num ||
 2435     reg ==  R7_num || reg == R7_H_num ||
 2436     reg ==  V0_num || reg == V0_H_num ||
 2437     reg ==  V1_num || reg == V1_H_num ||
 2438     reg ==  V2_num || reg == V2_H_num ||
 2439     reg ==  V3_num || reg == V3_H_num ||
 2440     reg ==  V4_num || reg == V4_H_num ||
 2441     reg ==  V5_num || reg == V5_H_num ||
 2442     reg ==  V6_num || reg == V6_H_num ||
 2443     reg ==  V7_num || reg == V7_H_num;
 2444 }
 2445 
 2446 bool Matcher::is_spillable_arg(int reg)
 2447 {
 2448   return can_be_java_arg(reg);
 2449 }
 2450 
 2451 uint Matcher::int_pressure_limit()
 2452 {
 2453   // JDK-8183543: When taking the number of available registers as int
 2454   // register pressure threshold, the jtreg test:
 2455   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2456   // failed due to C2 compilation failure with
 2457   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2458   //
 2459   // A derived pointer is live at CallNode and then is flagged by RA
 2460   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2461   // derived pointers and lastly fail to spill after reaching maximum
 2462   // number of iterations. Lowering the default pressure threshold to
 2463   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2464   // a high register pressure area of the code so that split_DEF can
 2465   // generate DefinitionSpillCopy for the derived pointer.
 2466   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2467   if (!PreserveFramePointer) {
 2468     // When PreserveFramePointer is off, frame pointer is allocatable,
 2469     // but different from other SOC registers, it is excluded from
 2470     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2471     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2472     // See check_pressure_at_fatproj().
 2473     default_int_pressure_threshold--;
 2474   }
 2475   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2476 }
 2477 
 2478 uint Matcher::float_pressure_limit()
 2479 {
 2480   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2481   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2482 }
 2483 
 2484 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2485   return false;
 2486 }
 2487 
 2488 RegMask Matcher::divI_proj_mask() {
 2489   ShouldNotReachHere();
 2490   return RegMask();
 2491 }
 2492 
 2493 // Register for MODI projection of divmodI.
 2494 RegMask Matcher::modI_proj_mask() {
 2495   ShouldNotReachHere();
 2496   return RegMask();
 2497 }
 2498 
 2499 // Register for DIVL projection of divmodL.
 2500 RegMask Matcher::divL_proj_mask() {
 2501   ShouldNotReachHere();
 2502   return RegMask();
 2503 }
 2504 
 2505 // Register for MODL projection of divmodL.
 2506 RegMask Matcher::modL_proj_mask() {
 2507   ShouldNotReachHere();
 2508   return RegMask();
 2509 }
 2510 
 2511 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2512   return FP_REG_mask();
 2513 }
 2514 
 2515 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2516   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2517     Node* u = addp->fast_out(i);
 2518     if (u->is_LoadStore()) {
 2519       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2520       // instructions) only take register indirect as an operand, so
 2521       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2522       // must fail.
 2523       return false;
 2524     }
 2525     if (u->is_Mem()) {
 2526       int opsize = u->as_Mem()->memory_size();
 2527       assert(opsize > 0, "unexpected memory operand size");
 2528       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2529         return false;
 2530       }
 2531     }
 2532   }
 2533   return true;
 2534 }
 2535 
 2536 // Binary src (Replicate con)
 2537 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2538   if (n == NULL || m == NULL) {
 2539     return false;
 2540   }
 2541 
 2542   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2543     return false;
 2544   }
 2545 
 2546   Node* imm_node = m->in(1);
 2547   if (!imm_node->is_Con()) {
 2548     return false;
 2549   }
 2550 
 2551   const Type* t = imm_node->bottom_type();
 2552   if (!(t->isa_int() || t->isa_long())) {
 2553     return false;
 2554   }
 2555 
 2556   switch (n->Opcode()) {
 2557   case Op_AndV:
 2558   case Op_OrV:
 2559   case Op_XorV: {
 2560     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2561     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2562     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2563   }
 2564   case Op_AddVB:
 2565     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2566   case Op_AddVS:
 2567   case Op_AddVI:
 2568     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2569   case Op_AddVL:
 2570     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2571   default:
 2572     return false;
 2573   }
 2574 }
 2575 
 2576 // (XorV src (Replicate m1))
 2577 // (XorVMask src (MaskAll m1))
 2578 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2579   if (n != NULL && m != NULL) {
 2580     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2581            VectorNode::is_all_ones_vector(m);
 2582   }
 2583   return false;
 2584 }
 2585 
 2586 // Should the matcher clone input 'm' of node 'n'?
 2587 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2588   if (is_vshift_con_pattern(n, m) ||
 2589       is_vector_bitwise_not_pattern(n, m) ||
 2590       is_valid_sve_arith_imm_pattern(n, m)) {
 2591     mstack.push(m, Visit);
 2592     return true;
 2593   }
 2594   return false;
 2595 }
 2596 
 2597 // Should the Matcher clone shifts on addressing modes, expecting them
 2598 // to be subsumed into complex addressing expressions or compute them
 2599 // into registers?
 2600 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2601   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2602     return true;
 2603   }
 2604 
 2605   Node *off = m->in(AddPNode::Offset);
 2606   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2607       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2608       // Are there other uses besides address expressions?
 2609       !is_visited(off)) {
 2610     address_visited.set(off->_idx); // Flag as address_visited
 2611     mstack.push(off->in(2), Visit);
 2612     Node *conv = off->in(1);
 2613     if (conv->Opcode() == Op_ConvI2L &&
 2614         // Are there other uses besides address expressions?
 2615         !is_visited(conv)) {
 2616       address_visited.set(conv->_idx); // Flag as address_visited
 2617       mstack.push(conv->in(1), Pre_Visit);
 2618     } else {
 2619       mstack.push(conv, Pre_Visit);
 2620     }
 2621     address_visited.test_set(m->_idx); // Flag as address_visited
 2622     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2623     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2624     return true;
 2625   } else if (off->Opcode() == Op_ConvI2L &&
 2626              // Are there other uses besides address expressions?
 2627              !is_visited(off)) {
 2628     address_visited.test_set(m->_idx); // Flag as address_visited
 2629     address_visited.set(off->_idx); // Flag as address_visited
 2630     mstack.push(off->in(1), Pre_Visit);
 2631     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2632     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2633     return true;
 2634   }
 2635   return false;
 2636 }
 2637 
 2638 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2639   C2_MacroAssembler _masm(&cbuf);                                       \
 2640   {                                                                     \
 2641     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2642     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2643     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2644     __ INSN(REG, as_Register(BASE));                                    \
 2645   }
 2646 
 2647 
 2648 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2649   {
 2650     Address::extend scale;
 2651 
 2652     // Hooboy, this is fugly.  We need a way to communicate to the
 2653     // encoder that the index needs to be sign extended, so we have to
 2654     // enumerate all the cases.
 2655     switch (opcode) {
 2656     case INDINDEXSCALEDI2L:
 2657     case INDINDEXSCALEDI2LN:
 2658     case INDINDEXI2L:
 2659     case INDINDEXI2LN:
 2660       scale = Address::sxtw(size);
 2661       break;
 2662     default:
 2663       scale = Address::lsl(size);
 2664     }
 2665 
 2666     if (index == -1) {
 2667       return Address(base, disp);
 2668     } else {
 2669       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2670       return Address(base, as_Register(index), scale);
 2671     }
 2672   }
 2673 
 2674 
 2675 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2676 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2677 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2678 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2679                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2680 
 2681   // Used for all non-volatile memory accesses.  The use of
 2682   // $mem->opcode() to discover whether this pattern uses sign-extended
 2683   // offsets is something of a kludge.
 2684   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2685                         Register reg, int opcode,
 2686                         Register base, int index, int scale, int disp,
 2687                         int size_in_memory)
 2688   {
 2689     Address addr = mem2address(opcode, base, index, scale, disp);
 2690     if (addr.getMode() == Address::base_plus_offset) {
 2691       /* If we get an out-of-range offset it is a bug in the compiler,
 2692          so we assert here. */
 2693       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2694              "c2 compiler bug");
 2695       /* Fix up any out-of-range offsets. */
 2696       assert_different_registers(rscratch1, base);
 2697       assert_different_registers(rscratch1, reg);
 2698       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2699     }
 2700     (masm.*insn)(reg, addr);
 2701   }
 2702 
 2703   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2704                         FloatRegister reg, int opcode,
 2705                         Register base, int index, int size, int disp,
 2706                         int size_in_memory)
 2707   {
 2708     Address::extend scale;
 2709 
 2710     switch (opcode) {
 2711     case INDINDEXSCALEDI2L:
 2712     case INDINDEXSCALEDI2LN:
 2713       scale = Address::sxtw(size);
 2714       break;
 2715     default:
 2716       scale = Address::lsl(size);
 2717     }
 2718 
 2719     if (index == -1) {
 2720       /* If we get an out-of-range offset it is a bug in the compiler,
 2721          so we assert here. */
 2722       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2723       /* Fix up any out-of-range offsets. */
 2724       assert_different_registers(rscratch1, base);
 2725       Address addr = Address(base, disp);
 2726       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2727       (masm.*insn)(reg, addr);
 2728     } else {
 2729       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2730       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2731     }
 2732   }
 2733 
 2734   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2735                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2736                         int opcode, Register base, int index, int size, int disp)
 2737   {
 2738     if (index == -1) {
 2739       (masm.*insn)(reg, T, Address(base, disp));
 2740     } else {
 2741       assert(disp == 0, "unsupported address mode");
 2742       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2743     }
 2744   }
 2745 
 2746 %}
 2747 
 2748 
 2749 
 2750 //----------ENCODING BLOCK-----------------------------------------------------
 2751 // This block specifies the encoding classes used by the compiler to
 2752 // output byte streams.  Encoding classes are parameterized macros
 2753 // used by Machine Instruction Nodes in order to generate the bit
 2754 // encoding of the instruction.  Operands specify their base encoding
 2755 // interface with the interface keyword.  There are currently
 2756 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2757 // COND_INTER.  REG_INTER causes an operand to generate a function
 2758 // which returns its register number when queried.  CONST_INTER causes
 2759 // an operand to generate a function which returns the value of the
 2760 // constant when queried.  MEMORY_INTER causes an operand to generate
 2761 // four functions which return the Base Register, the Index Register,
 2762 // the Scale Value, and the Offset Value of the operand when queried.
 2763 // COND_INTER causes an operand to generate six functions which return
 2764 // the encoding code (ie - encoding bits for the instruction)
 2765 // associated with each basic boolean condition for a conditional
 2766 // instruction.
 2767 //
 2768 // Instructions specify two basic values for encoding.  Again, a
 2769 // function is available to check if the constant displacement is an
 2770 // oop. They use the ins_encode keyword to specify their encoding
 2771 // classes (which must be a sequence of enc_class names, and their
 2772 // parameters, specified in the encoding block), and they use the
 2773 // opcode keyword to specify, in order, their primary, secondary, and
 2774 // tertiary opcode.  Only the opcode sections which a particular
 2775 // instruction needs for encoding need to be specified.
 2776 encode %{
 2777   // Build emit functions for each basic byte or larger field in the
 2778   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2779   // from C++ code in the enc_class source block.  Emit functions will
 2780   // live in the main source block for now.  In future, we can
 2781   // generalize this by adding a syntax that specifies the sizes of
 2782   // fields in an order, so that the adlc can build the emit functions
 2783   // automagically
 2784 
 2785   // catch all for unimplemented encodings
 2786   enc_class enc_unimplemented %{
 2787     C2_MacroAssembler _masm(&cbuf);
 2788     __ unimplemented("C2 catch all");
 2789   %}
 2790 
 2791   // BEGIN Non-volatile memory access
 2792 
 2793   // This encoding class is generated automatically from ad_encode.m4.
 2794   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2795   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2796     Register dst_reg = as_Register($dst$$reg);
 2797     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2798                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2799   %}
 2800 
 2801   // This encoding class is generated automatically from ad_encode.m4.
 2802   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2803   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2804     Register dst_reg = as_Register($dst$$reg);
 2805     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2806                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2807   %}
 2808 
 2809   // This encoding class is generated automatically from ad_encode.m4.
 2810   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2811   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2812     Register dst_reg = as_Register($dst$$reg);
 2813     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2814                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2815   %}
 2816 
 2817   // This encoding class is generated automatically from ad_encode.m4.
 2818   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2819   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2820     Register dst_reg = as_Register($dst$$reg);
 2821     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2822                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2823   %}
 2824 
 2825   // This encoding class is generated automatically from ad_encode.m4.
 2826   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2827   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2828     Register dst_reg = as_Register($dst$$reg);
 2829     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2830                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2831   %}
 2832 
 2833   // This encoding class is generated automatically from ad_encode.m4.
 2834   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2835   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2836     Register dst_reg = as_Register($dst$$reg);
 2837     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2838                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2839   %}
 2840 
 2841   // This encoding class is generated automatically from ad_encode.m4.
 2842   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2843   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2844     Register dst_reg = as_Register($dst$$reg);
 2845     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2846                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2847   %}
 2848 
 2849   // This encoding class is generated automatically from ad_encode.m4.
 2850   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2851   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2852     Register dst_reg = as_Register($dst$$reg);
 2853     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2854                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2855   %}
 2856 
 2857   // This encoding class is generated automatically from ad_encode.m4.
 2858   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2859   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2860     Register dst_reg = as_Register($dst$$reg);
 2861     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2862                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2863   %}
 2864 
 2865   // This encoding class is generated automatically from ad_encode.m4.
 2866   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2867   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2868     Register dst_reg = as_Register($dst$$reg);
 2869     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2870                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2871   %}
 2872 
 2873   // This encoding class is generated automatically from ad_encode.m4.
 2874   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2875   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2876     Register dst_reg = as_Register($dst$$reg);
 2877     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2878                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2879   %}
 2880 
 2881   // This encoding class is generated automatically from ad_encode.m4.
 2882   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2883   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2884     Register dst_reg = as_Register($dst$$reg);
 2885     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2886                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2887   %}
 2888 
 2889   // This encoding class is generated automatically from ad_encode.m4.
 2890   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2891   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2892     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2893     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2894                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2895   %}
 2896 
 2897   // This encoding class is generated automatically from ad_encode.m4.
 2898   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2899   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2900     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2901     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2902                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2903   %}
 2904 
 2905   // This encoding class is generated automatically from ad_encode.m4.
 2906   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2907   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2908     Register src_reg = as_Register($src$$reg);
 2909     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2910                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2911   %}
 2912 
 2913   // This encoding class is generated automatically from ad_encode.m4.
 2914   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2915   enc_class aarch64_enc_strb0(memory1 mem) %{
 2916     C2_MacroAssembler _masm(&cbuf);
 2917     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2918                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2919   %}
 2920 
 2921   // This encoding class is generated automatically from ad_encode.m4.
 2922   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2923   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2924     Register src_reg = as_Register($src$$reg);
 2925     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2926                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2927   %}
 2928 
 2929   // This encoding class is generated automatically from ad_encode.m4.
 2930   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2931   enc_class aarch64_enc_strh0(memory2 mem) %{
 2932     C2_MacroAssembler _masm(&cbuf);
 2933     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2934                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2935   %}
 2936 
 2937   // This encoding class is generated automatically from ad_encode.m4.
 2938   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2939   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2940     Register src_reg = as_Register($src$$reg);
 2941     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2942                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2943   %}
 2944 
 2945   // This encoding class is generated automatically from ad_encode.m4.
 2946   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2947   enc_class aarch64_enc_strw0(memory4 mem) %{
 2948     C2_MacroAssembler _masm(&cbuf);
 2949     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2950                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2951   %}
 2952 
 2953   // This encoding class is generated automatically from ad_encode.m4.
 2954   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2955   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2956     Register src_reg = as_Register($src$$reg);
 2957     // we sometimes get asked to store the stack pointer into the
 2958     // current thread -- we cannot do that directly on AArch64
 2959     if (src_reg == r31_sp) {
 2960       C2_MacroAssembler _masm(&cbuf);
 2961       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2962       __ mov(rscratch2, sp);
 2963       src_reg = rscratch2;
 2964     }
 2965     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2967   %}
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_str0(memory8 mem) %{
 2972     C2_MacroAssembler _masm(&cbuf);
 2973     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2980     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2981     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2988     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2989     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2991   %}
 2992 
 2993   // This encoding class is generated automatically from ad_encode.m4.
 2994   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2995   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 2996       C2_MacroAssembler _masm(&cbuf);
 2997       __ membar(Assembler::StoreStore);
 2998       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2999                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3000   %}
 3001 
 3002   // END Non-volatile memory access
 3003 
 3004   // Vector loads and stores
 3005   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3006     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3007     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3008        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3009   %}
 3010 
 3011   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3012     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3013     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3014        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3015   %}
 3016 
 3017   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3018     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3019     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3020        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3021   %}
 3022 
 3023   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3024     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3025     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3026        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3027   %}
 3028 
 3029   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3030     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3032        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3033   %}
 3034 
 3035   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3036     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3037     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3038        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3039   %}
 3040 
 3041   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3042     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3043     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3044        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3045   %}
 3046 
 3047   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3048     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3049     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3050        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3051   %}
 3052 
 3053   // volatile loads and stores
 3054 
 3055   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3056     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3057                  rscratch1, stlrb);
 3058   %}
 3059 
 3060   enc_class aarch64_enc_stlrb0(memory mem) %{
 3061     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3062                  rscratch1, stlrb);
 3063   %}
 3064 
 3065   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3066     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3067                  rscratch1, stlrh);
 3068   %}
 3069 
 3070   enc_class aarch64_enc_stlrh0(memory mem) %{
 3071     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3072                  rscratch1, stlrh);
 3073   %}
 3074 
 3075   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3076     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3077                  rscratch1, stlrw);
 3078   %}
 3079 
 3080   enc_class aarch64_enc_stlrw0(memory mem) %{
 3081     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3082                  rscratch1, stlrw);
 3083   %}
 3084 
 3085   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3086     Register dst_reg = as_Register($dst$$reg);
 3087     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3088              rscratch1, ldarb);
 3089     __ sxtbw(dst_reg, dst_reg);
 3090   %}
 3091 
 3092   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3093     Register dst_reg = as_Register($dst$$reg);
 3094     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3095              rscratch1, ldarb);
 3096     __ sxtb(dst_reg, dst_reg);
 3097   %}
 3098 
 3099   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3100     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3101              rscratch1, ldarb);
 3102   %}
 3103 
 3104   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3105     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3106              rscratch1, ldarb);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3110     Register dst_reg = as_Register($dst$$reg);
 3111     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3112              rscratch1, ldarh);
 3113     __ sxthw(dst_reg, dst_reg);
 3114   %}
 3115 
 3116   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3117     Register dst_reg = as_Register($dst$$reg);
 3118     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3119              rscratch1, ldarh);
 3120     __ sxth(dst_reg, dst_reg);
 3121   %}
 3122 
 3123   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3124     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3125              rscratch1, ldarh);
 3126   %}
 3127 
 3128   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3129     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3130              rscratch1, ldarh);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3134     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3135              rscratch1, ldarw);
 3136   %}
 3137 
 3138   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3139     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3140              rscratch1, ldarw);
 3141   %}
 3142 
 3143   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3144     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3145              rscratch1, ldar);
 3146   %}
 3147 
 3148   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3149     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3150              rscratch1, ldarw);
 3151     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3155     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3156              rscratch1, ldar);
 3157     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3158   %}
 3159 
 3160   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3161     Register src_reg = as_Register($src$$reg);
 3162     // we sometimes get asked to store the stack pointer into the
 3163     // current thread -- we cannot do that directly on AArch64
 3164     if (src_reg == r31_sp) {
 3165       C2_MacroAssembler _masm(&cbuf);
 3166       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3167       __ mov(rscratch2, sp);
 3168       src_reg = rscratch2;
 3169     }
 3170     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171                  rscratch1, stlr);
 3172   %}
 3173 
 3174   enc_class aarch64_enc_stlr0(memory mem) %{
 3175     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3176                  rscratch1, stlr);
 3177   %}
 3178 
 3179   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3180     {
 3181       C2_MacroAssembler _masm(&cbuf);
 3182       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3183       __ fmovs(rscratch2, src_reg);
 3184     }
 3185     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3186                  rscratch1, stlrw);
 3187   %}
 3188 
 3189   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3190     {
 3191       C2_MacroAssembler _masm(&cbuf);
 3192       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3193       __ fmovd(rscratch2, src_reg);
 3194     }
 3195     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3196                  rscratch1, stlr);
 3197   %}
 3198 
 3199   // synchronized read/update encodings
 3200 
 3201   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3202     C2_MacroAssembler _masm(&cbuf);
 3203     Register dst_reg = as_Register($dst$$reg);
 3204     Register base = as_Register($mem$$base);
 3205     int index = $mem$$index;
 3206     int scale = $mem$$scale;
 3207     int disp = $mem$$disp;
 3208     if (index == -1) {
 3209        if (disp != 0) {
 3210         __ lea(rscratch1, Address(base, disp));
 3211         __ ldaxr(dst_reg, rscratch1);
 3212       } else {
 3213         // TODO
 3214         // should we ever get anything other than this case?
 3215         __ ldaxr(dst_reg, base);
 3216       }
 3217     } else {
 3218       Register index_reg = as_Register(index);
 3219       if (disp == 0) {
 3220         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3221         __ ldaxr(dst_reg, rscratch1);
 3222       } else {
 3223         __ lea(rscratch1, Address(base, disp));
 3224         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3225         __ ldaxr(dst_reg, rscratch1);
 3226       }
 3227     }
 3228   %}
 3229 
 3230   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3231     C2_MacroAssembler _masm(&cbuf);
 3232     Register src_reg = as_Register($src$$reg);
 3233     Register base = as_Register($mem$$base);
 3234     int index = $mem$$index;
 3235     int scale = $mem$$scale;
 3236     int disp = $mem$$disp;
 3237     if (index == -1) {
 3238        if (disp != 0) {
 3239         __ lea(rscratch2, Address(base, disp));
 3240         __ stlxr(rscratch1, src_reg, rscratch2);
 3241       } else {
 3242         // TODO
 3243         // should we ever get anything other than this case?
 3244         __ stlxr(rscratch1, src_reg, base);
 3245       }
 3246     } else {
 3247       Register index_reg = as_Register(index);
 3248       if (disp == 0) {
 3249         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3250         __ stlxr(rscratch1, src_reg, rscratch2);
 3251       } else {
 3252         __ lea(rscratch2, Address(base, disp));
 3253         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3254         __ stlxr(rscratch1, src_reg, rscratch2);
 3255       }
 3256     }
 3257     __ cmpw(rscratch1, zr);
 3258   %}
 3259 
 3260   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3261     C2_MacroAssembler _masm(&cbuf);
 3262     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3263     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3264                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3265                /*weak*/ false, noreg);
 3266   %}
 3267 
 3268   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3269     C2_MacroAssembler _masm(&cbuf);
 3270     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3271     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3272                Assembler::word, /*acquire*/ false, /*release*/ true,
 3273                /*weak*/ false, noreg);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3277     C2_MacroAssembler _masm(&cbuf);
 3278     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3279     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3280                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3281                /*weak*/ false, noreg);
 3282   %}
 3283 
 3284   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3285     C2_MacroAssembler _masm(&cbuf);
 3286     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3287     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3288                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3289                /*weak*/ false, noreg);
 3290   %}
 3291 
 3292 
 3293   // The only difference between aarch64_enc_cmpxchg and
 3294   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3295   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3296   // lock.
 3297   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3298     C2_MacroAssembler _masm(&cbuf);
 3299     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3300     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3301                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3302                /*weak*/ false, noreg);
 3303   %}
 3304 
 3305   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3306     C2_MacroAssembler _masm(&cbuf);
 3307     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3308     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3309                Assembler::word, /*acquire*/ true, /*release*/ true,
 3310                /*weak*/ false, noreg);
 3311   %}
 3312 
 3313   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3314     C2_MacroAssembler _masm(&cbuf);
 3315     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3316     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3317                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3318                /*weak*/ false, noreg);
 3319   %}
 3320 
 3321   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3322     C2_MacroAssembler _masm(&cbuf);
 3323     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3324     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3325                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3326                /*weak*/ false, noreg);
 3327   %}
 3328 
 3329   // auxiliary used for CompareAndSwapX to set result register
 3330   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3331     C2_MacroAssembler _masm(&cbuf);
 3332     Register res_reg = as_Register($res$$reg);
 3333     __ cset(res_reg, Assembler::EQ);
 3334   %}
 3335 
 3336   // prefetch encodings
 3337 
 3338   enc_class aarch64_enc_prefetchw(memory mem) %{
 3339     C2_MacroAssembler _masm(&cbuf);
 3340     Register base = as_Register($mem$$base);
 3341     int index = $mem$$index;
 3342     int scale = $mem$$scale;
 3343     int disp = $mem$$disp;
 3344     if (index == -1) {
 3345       __ prfm(Address(base, disp), PSTL1KEEP);
 3346     } else {
 3347       Register index_reg = as_Register(index);
 3348       if (disp == 0) {
 3349         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3350       } else {
 3351         __ lea(rscratch1, Address(base, disp));
 3352 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3353       }
 3354     }
 3355   %}
 3356 
 3357   /// mov envcodings
 3358 
 3359   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3360     C2_MacroAssembler _masm(&cbuf);
 3361     uint32_t con = (uint32_t)$src$$constant;
 3362     Register dst_reg = as_Register($dst$$reg);
 3363     if (con == 0) {
 3364       __ movw(dst_reg, zr);
 3365     } else {
 3366       __ movw(dst_reg, con);
 3367     }
 3368   %}
 3369 
 3370   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3371     C2_MacroAssembler _masm(&cbuf);
 3372     Register dst_reg = as_Register($dst$$reg);
 3373     uint64_t con = (uint64_t)$src$$constant;
 3374     if (con == 0) {
 3375       __ mov(dst_reg, zr);
 3376     } else {
 3377       __ mov(dst_reg, con);
 3378     }
 3379   %}
 3380 
 3381   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3382     C2_MacroAssembler _masm(&cbuf);
 3383     Register dst_reg = as_Register($dst$$reg);
 3384     address con = (address)$src$$constant;
 3385     if (con == NULL || con == (address)1) {
 3386       ShouldNotReachHere();
 3387     } else {
 3388       relocInfo::relocType rtype = $src->constant_reloc();
 3389       if (rtype == relocInfo::oop_type) {
 3390         __ movoop(dst_reg, (jobject)con);
 3391       } else if (rtype == relocInfo::metadata_type) {
 3392         __ mov_metadata(dst_reg, (Metadata*)con);
 3393       } else {
 3394         assert(rtype == relocInfo::none, "unexpected reloc type");
 3395         if (! __ is_valid_AArch64_address(con) ||
 3396             con < (address)(uintptr_t)os::vm_page_size()) {
 3397           __ mov(dst_reg, con);
 3398         } else {
 3399           uint64_t offset;
 3400           __ adrp(dst_reg, con, offset);
 3401           __ add(dst_reg, dst_reg, offset);
 3402         }
 3403       }
 3404     }
 3405   %}
 3406 
 3407   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3408     C2_MacroAssembler _masm(&cbuf);
 3409     Register dst_reg = as_Register($dst$$reg);
 3410     __ mov(dst_reg, zr);
 3411   %}
 3412 
 3413   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3414     C2_MacroAssembler _masm(&cbuf);
 3415     Register dst_reg = as_Register($dst$$reg);
 3416     __ mov(dst_reg, (uint64_t)1);
 3417   %}
 3418 
 3419   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3420     C2_MacroAssembler _masm(&cbuf);
 3421     __ load_byte_map_base($dst$$Register);
 3422   %}
 3423 
 3424   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3425     C2_MacroAssembler _masm(&cbuf);
 3426     Register dst_reg = as_Register($dst$$reg);
 3427     address con = (address)$src$$constant;
 3428     if (con == NULL) {
 3429       ShouldNotReachHere();
 3430     } else {
 3431       relocInfo::relocType rtype = $src->constant_reloc();
 3432       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3433       __ set_narrow_oop(dst_reg, (jobject)con);
 3434     }
 3435   %}
 3436 
 3437   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3438     C2_MacroAssembler _masm(&cbuf);
 3439     Register dst_reg = as_Register($dst$$reg);
 3440     __ mov(dst_reg, zr);
 3441   %}
 3442 
 3443   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3444     C2_MacroAssembler _masm(&cbuf);
 3445     Register dst_reg = as_Register($dst$$reg);
 3446     address con = (address)$src$$constant;
 3447     if (con == NULL) {
 3448       ShouldNotReachHere();
 3449     } else {
 3450       relocInfo::relocType rtype = $src->constant_reloc();
 3451       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3452       __ set_narrow_klass(dst_reg, (Klass *)con);
 3453     }
 3454   %}
 3455 
 3456   // arithmetic encodings
 3457 
 3458   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3459     C2_MacroAssembler _masm(&cbuf);
 3460     Register dst_reg = as_Register($dst$$reg);
 3461     Register src_reg = as_Register($src1$$reg);
 3462     int32_t con = (int32_t)$src2$$constant;
 3463     // add has primary == 0, subtract has primary == 1
 3464     if ($primary) { con = -con; }
 3465     if (con < 0) {
 3466       __ subw(dst_reg, src_reg, -con);
 3467     } else {
 3468       __ addw(dst_reg, src_reg, con);
 3469     }
 3470   %}
 3471 
 3472   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3473     C2_MacroAssembler _masm(&cbuf);
 3474     Register dst_reg = as_Register($dst$$reg);
 3475     Register src_reg = as_Register($src1$$reg);
 3476     int32_t con = (int32_t)$src2$$constant;
 3477     // add has primary == 0, subtract has primary == 1
 3478     if ($primary) { con = -con; }
 3479     if (con < 0) {
 3480       __ sub(dst_reg, src_reg, -con);
 3481     } else {
 3482       __ add(dst_reg, src_reg, con);
 3483     }
 3484   %}
 3485 
 3486   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3487     C2_MacroAssembler _masm(&cbuf);
 3488    Register dst_reg = as_Register($dst$$reg);
 3489    Register src1_reg = as_Register($src1$$reg);
 3490    Register src2_reg = as_Register($src2$$reg);
 3491     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3492   %}
 3493 
 3494   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3495     C2_MacroAssembler _masm(&cbuf);
 3496    Register dst_reg = as_Register($dst$$reg);
 3497    Register src1_reg = as_Register($src1$$reg);
 3498    Register src2_reg = as_Register($src2$$reg);
 3499     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3500   %}
 3501 
 3502   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3503     C2_MacroAssembler _masm(&cbuf);
 3504    Register dst_reg = as_Register($dst$$reg);
 3505    Register src1_reg = as_Register($src1$$reg);
 3506    Register src2_reg = as_Register($src2$$reg);
 3507     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3508   %}
 3509 
 3510   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3511     C2_MacroAssembler _masm(&cbuf);
 3512    Register dst_reg = as_Register($dst$$reg);
 3513    Register src1_reg = as_Register($src1$$reg);
 3514    Register src2_reg = as_Register($src2$$reg);
 3515     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3516   %}
 3517 
 3518   // compare instruction encodings
 3519 
 3520   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3521     C2_MacroAssembler _masm(&cbuf);
 3522     Register reg1 = as_Register($src1$$reg);
 3523     Register reg2 = as_Register($src2$$reg);
 3524     __ cmpw(reg1, reg2);
 3525   %}
 3526 
 3527   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3528     C2_MacroAssembler _masm(&cbuf);
 3529     Register reg = as_Register($src1$$reg);
 3530     int32_t val = $src2$$constant;
 3531     if (val >= 0) {
 3532       __ subsw(zr, reg, val);
 3533     } else {
 3534       __ addsw(zr, reg, -val);
 3535     }
 3536   %}
 3537 
 3538   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3539     C2_MacroAssembler _masm(&cbuf);
 3540     Register reg1 = as_Register($src1$$reg);
 3541     uint32_t val = (uint32_t)$src2$$constant;
 3542     __ movw(rscratch1, val);
 3543     __ cmpw(reg1, rscratch1);
 3544   %}
 3545 
 3546   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3547     C2_MacroAssembler _masm(&cbuf);
 3548     Register reg1 = as_Register($src1$$reg);
 3549     Register reg2 = as_Register($src2$$reg);
 3550     __ cmp(reg1, reg2);
 3551   %}
 3552 
 3553   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3554     C2_MacroAssembler _masm(&cbuf);
 3555     Register reg = as_Register($src1$$reg);
 3556     int64_t val = $src2$$constant;
 3557     if (val >= 0) {
 3558       __ subs(zr, reg, val);
 3559     } else if (val != -val) {
 3560       __ adds(zr, reg, -val);
 3561     } else {
 3562     // aargh, Long.MIN_VALUE is a special case
 3563       __ orr(rscratch1, zr, (uint64_t)val);
 3564       __ subs(zr, reg, rscratch1);
 3565     }
 3566   %}
 3567 
 3568   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3569     C2_MacroAssembler _masm(&cbuf);
 3570     Register reg1 = as_Register($src1$$reg);
 3571     uint64_t val = (uint64_t)$src2$$constant;
 3572     __ mov(rscratch1, val);
 3573     __ cmp(reg1, rscratch1);
 3574   %}
 3575 
 3576   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3577     C2_MacroAssembler _masm(&cbuf);
 3578     Register reg1 = as_Register($src1$$reg);
 3579     Register reg2 = as_Register($src2$$reg);
 3580     __ cmp(reg1, reg2);
 3581   %}
 3582 
 3583   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3584     C2_MacroAssembler _masm(&cbuf);
 3585     Register reg1 = as_Register($src1$$reg);
 3586     Register reg2 = as_Register($src2$$reg);
 3587     __ cmpw(reg1, reg2);
 3588   %}
 3589 
 3590   enc_class aarch64_enc_testp(iRegP src) %{
 3591     C2_MacroAssembler _masm(&cbuf);
 3592     Register reg = as_Register($src$$reg);
 3593     __ cmp(reg, zr);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_testn(iRegN src) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598     Register reg = as_Register($src$$reg);
 3599     __ cmpw(reg, zr);
 3600   %}
 3601 
 3602   enc_class aarch64_enc_b(label lbl) %{
 3603     C2_MacroAssembler _masm(&cbuf);
 3604     Label *L = $lbl$$label;
 3605     __ b(*L);
 3606   %}
 3607 
 3608   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3609     C2_MacroAssembler _masm(&cbuf);
 3610     Label *L = $lbl$$label;
 3611     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3612   %}
 3613 
 3614   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3615     C2_MacroAssembler _masm(&cbuf);
 3616     Label *L = $lbl$$label;
 3617     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3618   %}
 3619 
 3620   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3621   %{
 3622      Register sub_reg = as_Register($sub$$reg);
 3623      Register super_reg = as_Register($super$$reg);
 3624      Register temp_reg = as_Register($temp$$reg);
 3625      Register result_reg = as_Register($result$$reg);
 3626 
 3627      Label miss;
 3628      C2_MacroAssembler _masm(&cbuf);
 3629      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3630                                      NULL, &miss,
 3631                                      /*set_cond_codes:*/ true);
 3632      if ($primary) {
 3633        __ mov(result_reg, zr);
 3634      }
 3635      __ bind(miss);
 3636   %}
 3637 
 3638   enc_class aarch64_enc_java_static_call(method meth) %{
 3639     C2_MacroAssembler _masm(&cbuf);
 3640 
 3641     address addr = (address)$meth$$method;
 3642     address call;
 3643     if (!_method) {
 3644       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3645       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3646       if (call == NULL) {
 3647         ciEnv::current()->record_failure("CodeCache is full");
 3648         return;
 3649       }
 3650     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3651       // The NOP here is purely to ensure that eliding a call to
 3652       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3653       __ nop();
 3654       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3655     } else {
 3656       int method_index = resolved_method_index(cbuf);
 3657       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3658                                                   : static_call_Relocation::spec(method_index);
 3659       call = __ trampoline_call(Address(addr, rspec));
 3660       if (call == NULL) {
 3661         ciEnv::current()->record_failure("CodeCache is full");
 3662         return;
 3663       }
 3664       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3665         // Calls of the same statically bound method can share
 3666         // a stub to the interpreter.
 3667         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3668       } else {
 3669         // Emit stub for static call
 3670         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3671         if (stub == NULL) {
 3672           ciEnv::current()->record_failure("CodeCache is full");
 3673           return;
 3674         }
 3675       }
 3676     }
 3677 
 3678     __ post_call_nop();
 3679 
 3680     // Only non uncommon_trap calls need to reinitialize ptrue.
 3681     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3682       __ reinitialize_ptrue();
 3683     }
 3684   %}
 3685 
 3686   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3687     C2_MacroAssembler _masm(&cbuf);
 3688     int method_index = resolved_method_index(cbuf);
 3689     address call = __ ic_call((address)$meth$$method, method_index);
 3690     if (call == NULL) {
 3691       ciEnv::current()->record_failure("CodeCache is full");
 3692       return;
 3693     }
 3694     __ post_call_nop();
 3695     if (Compile::current()->max_vector_size() > 0) {
 3696       __ reinitialize_ptrue();
 3697     }
 3698   %}
 3699 
 3700   enc_class aarch64_enc_call_epilog() %{
 3701     C2_MacroAssembler _masm(&cbuf);
 3702     if (VerifyStackAtCalls) {
 3703       // Check that stack depth is unchanged: find majik cookie on stack
 3704       __ call_Unimplemented();
 3705     }
 3706   %}
 3707 
 3708   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3709     C2_MacroAssembler _masm(&cbuf);
 3710 
 3711     // some calls to generated routines (arraycopy code) are scheduled
 3712     // by C2 as runtime calls. if so we can call them using a br (they
 3713     // will be in a reachable segment) otherwise we have to use a blr
 3714     // which loads the absolute address into a register.
 3715     address entry = (address)$meth$$method;
 3716     CodeBlob *cb = CodeCache::find_blob(entry);
 3717     if (cb) {
 3718       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3719       if (call == NULL) {
 3720         ciEnv::current()->record_failure("CodeCache is full");
 3721         return;
 3722       }
 3723       __ post_call_nop();
 3724     } else {
 3725       Label retaddr;
 3726       __ adr(rscratch2, retaddr);
 3727       __ lea(rscratch1, RuntimeAddress(entry));
 3728       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3729       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3730       __ blr(rscratch1);
 3731       __ bind(retaddr);
 3732       __ post_call_nop();
 3733       __ add(sp, sp, 2 * wordSize);
 3734     }
 3735     if (Compile::current()->max_vector_size() > 0) {
 3736       __ reinitialize_ptrue();
 3737     }
 3738   %}
 3739 
 3740   enc_class aarch64_enc_rethrow() %{
 3741     C2_MacroAssembler _masm(&cbuf);
 3742     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3743   %}
 3744 
 3745   enc_class aarch64_enc_ret() %{
 3746     C2_MacroAssembler _masm(&cbuf);
 3747 #ifdef ASSERT
 3748     if (Compile::current()->max_vector_size() > 0) {
 3749       __ verify_ptrue();
 3750     }
 3751 #endif
 3752     __ ret(lr);
 3753   %}
 3754 
 3755   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3756     C2_MacroAssembler _masm(&cbuf);
 3757     Register target_reg = as_Register($jump_target$$reg);
 3758     __ br(target_reg);
 3759   %}
 3760 
 3761   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3762     C2_MacroAssembler _masm(&cbuf);
 3763     Register target_reg = as_Register($jump_target$$reg);
 3764     // exception oop should be in r0
 3765     // ret addr has been popped into lr
 3766     // callee expects it in r3
 3767     __ mov(r3, lr);
 3768     __ br(target_reg);
 3769   %}
 3770 
 3771   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3772     C2_MacroAssembler _masm(&cbuf);
 3773     Register oop = as_Register($object$$reg);
 3774     Register box = as_Register($box$$reg);
 3775     Register disp_hdr = as_Register($tmp$$reg);
 3776     Register tmp = as_Register($tmp2$$reg);
 3777     Label cont;
 3778     Label object_has_monitor;
 3779     Label count, no_count;
 3780 
 3781     assert_different_registers(oop, box, tmp, disp_hdr);
 3782 
 3783     // Load markWord from object into displaced_header.
 3784     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3785 
 3786     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3787       __ load_klass(tmp, oop);
 3788       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3789       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3790       __ br(Assembler::NE, cont);
 3791     }
 3792 
 3793     // Check for existing monitor
 3794     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3795 
 3796     if (!UseHeavyMonitors) {
 3797       if (UseFastLocking) {
 3798         __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count, false);












 3799 
 3800         // Indicate success at cont.
 3801         __ cmp(oop, oop);
 3802         __ b(count);
 3803       } else {
 3804         // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3805         __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3806 
 3807         // Initialize the box. (Must happen before we update the object mark!)
 3808         __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3809 
 3810         // Compare object markWord with an unlocked value (tmp) and if
 3811         // equal exchange the stack address of our box with object markWord.
 3812         // On failure disp_hdr contains the possibly locked markWord.
 3813         __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3814                    /*release*/ true, /*weak*/ false, disp_hdr);
 3815         __ br(Assembler::EQ, cont);
 3816 
 3817         assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3818 
 3819         // If the compare-and-exchange succeeded, then we found an unlocked
 3820         // object, will have now locked it will continue at label cont
 3821 
 3822         // Check if the owner is self by comparing the value in the
 3823         // markWord of object (disp_hdr) with the stack pointer.
 3824         __ mov(rscratch1, sp);
 3825         __ sub(disp_hdr, disp_hdr, rscratch1);
 3826         __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3827         // If condition is true we are cont and hence we can store 0 as the
 3828         // displaced header in the box, which indicates that it is a recursive lock.
 3829         __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3830         __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3831         __ b(cont);
 3832       }
 3833     } else {
 3834       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3835       __ b(cont);
 3836     }

 3837 
 3838     // Handle existing monitor.
 3839     __ bind(object_has_monitor);
 3840 
 3841     // The object's monitor m is unlocked iff m->owner == NULL,
 3842     // otherwise m->owner may contain a thread or a stack address.
 3843     //
 3844     // Try to CAS m->owner from NULL to current thread.
 3845     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3846     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3847                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3848 
 3849     if (!UseFastLocking) {
 3850       // Store a non-null value into the box to avoid looking like a re-entrant
 3851       // lock. The fast-path monitor unlock code checks for
 3852       // markWord::monitor_value so use markWord::unused_mark which has the
 3853       // relevant bit set, and also matches ObjectSynchronizer::enter.
 3854       __ mov(tmp, (address)markWord::unused_mark().value());
 3855       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3856     }
 3857     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3858 
 3859     __ cmp(rscratch1, rthread);
 3860     __ br(Assembler::NE, cont); // Check for recursive locking
 3861 
 3862     // Recursive lock case
 3863     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3864     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3865 
 3866     __ bind(cont);
 3867     // flag == EQ indicates success
 3868     // flag == NE indicates failure
 3869     __ br(Assembler::NE, no_count);
 3870 
 3871     __ bind(count);
 3872     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3873 
 3874     __ bind(no_count);
 3875   %}
 3876 
 3877   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3878     C2_MacroAssembler _masm(&cbuf);
 3879     Register oop = as_Register($object$$reg);
 3880     Register box = as_Register($box$$reg);
 3881     Register disp_hdr = as_Register($tmp$$reg);
 3882     Register tmp = as_Register($tmp2$$reg);
 3883     Label cont;
 3884     Label object_has_monitor;
 3885     Label count, no_count;
 3886 
 3887     assert_different_registers(oop, box, tmp, disp_hdr);
 3888 
 3889     if (!UseHeavyMonitors && !UseFastLocking) {
 3890       // Find the lock address and load the displaced header from the stack.
 3891       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3892 
 3893       // If the displaced header is 0, we have a recursive unlock.
 3894       __ cmp(disp_hdr, zr);
 3895       __ br(Assembler::EQ, cont);
 3896     }
 3897 
 3898     // Handle existing monitor.
 3899     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3900     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3901 
 3902     if (!UseHeavyMonitors) {
 3903       if (UseFastLocking) {
 3904         __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
 3905 
 3906         // Indicate success at cont.
 3907         __ cmp(oop, oop);
 3908         __ b(count);
 3909       } else {
 3910         // Check if it is still a light weight lock, this is is true if we
 3911         // see the stack address of the basicLock in the markWord of the
 3912         // object.
 3913 
 3914         __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3915                    /*release*/ true, /*weak*/ false, tmp);
 3916         __ b(cont);
 3917       }
 3918     } else {
 3919       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3920       __ b(cont);
 3921     }

 3922 
 3923     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3924 
 3925     // Handle existing monitor.
 3926     __ bind(object_has_monitor);
 3927     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3928     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3929 
 3930     if (UseFastLocking) {
 3931       // If the owner is anonymous, we need to fix it -- in the slow-path.
 3932       __ ldr(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3933       // We cannot use tbnz here: tbnz would leave the condition flags untouched,
 3934       // but we want to carry-over the NE condition to the exit at the cont label,
 3935       // in order to take the slow-path.
 3936       __ tst(disp_hdr, (uint64_t)(intptr_t) ANONYMOUS_OWNER);
 3937       __ br(Assembler::NE, no_count);
 3938     }
 3939 
 3940     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3941 
 3942     Label notRecursive;
 3943     __ cbz(disp_hdr, notRecursive);
 3944 
 3945     // Recursive lock
 3946     __ sub(disp_hdr, disp_hdr, 1u);
 3947     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3948     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3949     __ b(cont);
 3950 
 3951     __ bind(notRecursive);
 3952     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3953     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3954     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3955     __ cmp(rscratch1, zr); // Sets flags for result
 3956     __ cbnz(rscratch1, cont);
 3957     // need a release store here
 3958     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3959     __ stlr(zr, tmp); // set unowned
 3960 
 3961     __ bind(cont);
 3962     // flag == EQ indicates success
 3963     // flag == NE indicates failure
 3964     __ br(Assembler::NE, no_count);
 3965 
 3966     __ bind(count);
 3967     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3968 
 3969     __ bind(no_count);
 3970   %}
 3971 
 3972 %}
 3973 
 3974 //----------FRAME--------------------------------------------------------------
 3975 // Definition of frame structure and management information.
 3976 //
 3977 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3978 //                             |   (to get allocators register number
 3979 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3980 //  r   CALLER     |        |
 3981 //  o     |        +--------+      pad to even-align allocators stack-slot
 3982 //  w     V        |  pad0  |        numbers; owned by CALLER
 3983 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3984 //  h     ^        |   in   |  5
 3985 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3986 //  |     |        |        |  3
 3987 //  |     |        +--------+
 3988 //  V     |        | old out|      Empty on Intel, window on Sparc
 3989 //        |    old |preserve|      Must be even aligned.
 3990 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3991 //        |        |   in   |  3   area for Intel ret address
 3992 //     Owned by    |preserve|      Empty on Sparc.
 3993 //       SELF      +--------+
 3994 //        |        |  pad2  |  2   pad to align old SP
 3995 //        |        +--------+  1
 3996 //        |        | locks  |  0
 3997 //        |        +--------+----> OptoReg::stack0(), even aligned
 3998 //        |        |  pad1  | 11   pad to align new SP
 3999 //        |        +--------+
 4000 //        |        |        | 10
 4001 //        |        | spills |  9   spills
 4002 //        V        |        |  8   (pad0 slot for callee)
 4003 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 4004 //        ^        |  out   |  7
 4005 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4006 //     Owned by    +--------+
 4007 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4008 //        |    new |preserve|      Must be even-aligned.
 4009 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4010 //        |        |        |
 4011 //
 4012 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4013 //         known from SELF's arguments and the Java calling convention.
 4014 //         Region 6-7 is determined per call site.
 4015 // Note 2: If the calling convention leaves holes in the incoming argument
 4016 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4017 //         are owned by the CALLEE.  Holes should not be necessary in the
 4018 //         incoming area, as the Java calling convention is completely under
 4019 //         the control of the AD file.  Doubles can be sorted and packed to
 4020 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4021 //         varargs C calling conventions.
 4022 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4023 //         even aligned with pad0 as needed.
 4024 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4025 //           (the latter is true on Intel but is it false on AArch64?)
 4026 //         region 6-11 is even aligned; it may be padded out more so that
 4027 //         the region from SP to FP meets the minimum stack alignment.
 4028 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4029 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4030 //         SP meets the minimum alignment.
 4031 
 4032 frame %{
 4033   // These three registers define part of the calling convention
 4034   // between compiled code and the interpreter.
 4035 
 4036   // Inline Cache Register or Method for I2C.
 4037   inline_cache_reg(R12);
 4038 
 4039   // Number of stack slots consumed by locking an object
 4040   sync_stack_slots(2);
 4041 
 4042   // Compiled code's Frame Pointer
 4043   frame_pointer(R31);
 4044 
 4045   // Interpreter stores its frame pointer in a register which is
 4046   // stored to the stack by I2CAdaptors.
 4047   // I2CAdaptors convert from interpreted java to compiled java.
 4048   interpreter_frame_pointer(R29);
 4049 
 4050   // Stack alignment requirement
 4051   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4052 
 4053   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4054   // for calls to C.  Supports the var-args backing area for register parms.
 4055   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4056 
 4057   // The after-PROLOG location of the return address.  Location of
 4058   // return address specifies a type (REG or STACK) and a number
 4059   // representing the register number (i.e. - use a register name) or
 4060   // stack slot.
 4061   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4062   // Otherwise, it is above the locks and verification slot and alignment word
 4063   // TODO this may well be correct but need to check why that - 2 is there
 4064   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4065   // which folds in the space used for monitors
 4066   return_addr(STACK - 2 +
 4067               align_up((Compile::current()->in_preserve_stack_slots() +
 4068                         Compile::current()->fixed_slots()),
 4069                        stack_alignment_in_slots()));
 4070 
 4071   // Location of compiled Java return values.  Same as C for now.
 4072   return_value
 4073   %{
 4074     // TODO do we allow ideal_reg == Op_RegN???
 4075     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4076            "only return normal values");
 4077 
 4078     static const int lo[Op_RegL + 1] = { // enum name
 4079       0,                                 // Op_Node
 4080       0,                                 // Op_Set
 4081       R0_num,                            // Op_RegN
 4082       R0_num,                            // Op_RegI
 4083       R0_num,                            // Op_RegP
 4084       V0_num,                            // Op_RegF
 4085       V0_num,                            // Op_RegD
 4086       R0_num                             // Op_RegL
 4087     };
 4088 
 4089     static const int hi[Op_RegL + 1] = { // enum name
 4090       0,                                 // Op_Node
 4091       0,                                 // Op_Set
 4092       OptoReg::Bad,                      // Op_RegN
 4093       OptoReg::Bad,                      // Op_RegI
 4094       R0_H_num,                          // Op_RegP
 4095       OptoReg::Bad,                      // Op_RegF
 4096       V0_H_num,                          // Op_RegD
 4097       R0_H_num                           // Op_RegL
 4098     };
 4099 
 4100     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4101   %}
 4102 %}
 4103 
 4104 //----------ATTRIBUTES---------------------------------------------------------
 4105 //----------Operand Attributes-------------------------------------------------
 4106 op_attrib op_cost(1);        // Required cost attribute
 4107 
 4108 //----------Instruction Attributes---------------------------------------------
 4109 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4110 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4111 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4112                                 // a non-matching short branch variant
 4113                                 // of some long branch?
 4114 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4115                                 // be a power of 2) specifies the
 4116                                 // alignment that some part of the
 4117                                 // instruction (not necessarily the
 4118                                 // start) requires.  If > 1, a
 4119                                 // compute_padding() function must be
 4120                                 // provided for the instruction
 4121 
 4122 //----------OPERANDS-----------------------------------------------------------
 4123 // Operand definitions must precede instruction definitions for correct parsing
 4124 // in the ADLC because operands constitute user defined types which are used in
 4125 // instruction definitions.
 4126 
 4127 //----------Simple Operands----------------------------------------------------
 4128 
 4129 // Integer operands 32 bit
 4130 // 32 bit immediate
 4131 operand immI()
 4132 %{
 4133   match(ConI);
 4134 
 4135   op_cost(0);
 4136   format %{ %}
 4137   interface(CONST_INTER);
 4138 %}
 4139 
 4140 // 32 bit zero
 4141 operand immI0()
 4142 %{
 4143   predicate(n->get_int() == 0);
 4144   match(ConI);
 4145 
 4146   op_cost(0);
 4147   format %{ %}
 4148   interface(CONST_INTER);
 4149 %}
 4150 
 4151 // 32 bit unit increment
 4152 operand immI_1()
 4153 %{
 4154   predicate(n->get_int() == 1);
 4155   match(ConI);
 4156 
 4157   op_cost(0);
 4158   format %{ %}
 4159   interface(CONST_INTER);
 4160 %}
 4161 
 4162 // 32 bit unit decrement
 4163 operand immI_M1()
 4164 %{
 4165   predicate(n->get_int() == -1);
 4166   match(ConI);
 4167 
 4168   op_cost(0);
 4169   format %{ %}
 4170   interface(CONST_INTER);
 4171 %}
 4172 
 4173 // Shift values for add/sub extension shift
 4174 operand immIExt()
 4175 %{
 4176   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4177   match(ConI);
 4178 
 4179   op_cost(0);
 4180   format %{ %}
 4181   interface(CONST_INTER);
 4182 %}
 4183 
 4184 operand immI_gt_1()
 4185 %{
 4186   predicate(n->get_int() > 1);
 4187   match(ConI);
 4188 
 4189   op_cost(0);
 4190   format %{ %}
 4191   interface(CONST_INTER);
 4192 %}
 4193 
 4194 operand immI_le_4()
 4195 %{
 4196   predicate(n->get_int() <= 4);
 4197   match(ConI);
 4198 
 4199   op_cost(0);
 4200   format %{ %}
 4201   interface(CONST_INTER);
 4202 %}
 4203 
 4204 operand immI_16()
 4205 %{
 4206   predicate(n->get_int() == 16);
 4207   match(ConI);
 4208 
 4209   op_cost(0);
 4210   format %{ %}
 4211   interface(CONST_INTER);
 4212 %}
 4213 
 4214 operand immI_24()
 4215 %{
 4216   predicate(n->get_int() == 24);
 4217   match(ConI);
 4218 
 4219   op_cost(0);
 4220   format %{ %}
 4221   interface(CONST_INTER);
 4222 %}
 4223 
 4224 operand immI_32()
 4225 %{
 4226   predicate(n->get_int() == 32);
 4227   match(ConI);
 4228 
 4229   op_cost(0);
 4230   format %{ %}
 4231   interface(CONST_INTER);
 4232 %}
 4233 
 4234 operand immI_48()
 4235 %{
 4236   predicate(n->get_int() == 48);
 4237   match(ConI);
 4238 
 4239   op_cost(0);
 4240   format %{ %}
 4241   interface(CONST_INTER);
 4242 %}
 4243 
 4244 operand immI_56()
 4245 %{
 4246   predicate(n->get_int() == 56);
 4247   match(ConI);
 4248 
 4249   op_cost(0);
 4250   format %{ %}
 4251   interface(CONST_INTER);
 4252 %}
 4253 
 4254 operand immI_63()
 4255 %{
 4256   predicate(n->get_int() == 63);
 4257   match(ConI);
 4258 
 4259   op_cost(0);
 4260   format %{ %}
 4261   interface(CONST_INTER);
 4262 %}
 4263 
 4264 operand immI_64()
 4265 %{
 4266   predicate(n->get_int() == 64);
 4267   match(ConI);
 4268 
 4269   op_cost(0);
 4270   format %{ %}
 4271   interface(CONST_INTER);
 4272 %}
 4273 
 4274 operand immI_255()
 4275 %{
 4276   predicate(n->get_int() == 255);
 4277   match(ConI);
 4278 
 4279   op_cost(0);
 4280   format %{ %}
 4281   interface(CONST_INTER);
 4282 %}
 4283 
 4284 operand immI_65535()
 4285 %{
 4286   predicate(n->get_int() == 65535);
 4287   match(ConI);
 4288 
 4289   op_cost(0);
 4290   format %{ %}
 4291   interface(CONST_INTER);
 4292 %}
 4293 
 4294 operand immI_positive()
 4295 %{
 4296   predicate(n->get_int() > 0);
 4297   match(ConI);
 4298 
 4299   op_cost(0);
 4300   format %{ %}
 4301   interface(CONST_INTER);
 4302 %}
 4303 
 4304 operand immL_255()
 4305 %{
 4306   predicate(n->get_long() == 255L);
 4307   match(ConL);
 4308 
 4309   op_cost(0);
 4310   format %{ %}
 4311   interface(CONST_INTER);
 4312 %}
 4313 
 4314 operand immL_65535()
 4315 %{
 4316   predicate(n->get_long() == 65535L);
 4317   match(ConL);
 4318 
 4319   op_cost(0);
 4320   format %{ %}
 4321   interface(CONST_INTER);
 4322 %}
 4323 
 4324 operand immL_4294967295()
 4325 %{
 4326   predicate(n->get_long() == 4294967295L);
 4327   match(ConL);
 4328 
 4329   op_cost(0);
 4330   format %{ %}
 4331   interface(CONST_INTER);
 4332 %}
 4333 
 4334 operand immL_bitmask()
 4335 %{
 4336   predicate((n->get_long() != 0)
 4337             && ((n->get_long() & 0xc000000000000000l) == 0)
 4338             && is_power_of_2(n->get_long() + 1));
 4339   match(ConL);
 4340 
 4341   op_cost(0);
 4342   format %{ %}
 4343   interface(CONST_INTER);
 4344 %}
 4345 
 4346 operand immI_bitmask()
 4347 %{
 4348   predicate((n->get_int() != 0)
 4349             && ((n->get_int() & 0xc0000000) == 0)
 4350             && is_power_of_2(n->get_int() + 1));
 4351   match(ConI);
 4352 
 4353   op_cost(0);
 4354   format %{ %}
 4355   interface(CONST_INTER);
 4356 %}
 4357 
 4358 operand immL_positive_bitmaskI()
 4359 %{
 4360   predicate((n->get_long() != 0)
 4361             && ((julong)n->get_long() < 0x80000000ULL)
 4362             && is_power_of_2(n->get_long() + 1));
 4363   match(ConL);
 4364 
 4365   op_cost(0);
 4366   format %{ %}
 4367   interface(CONST_INTER);
 4368 %}
 4369 
 4370 // Scale values for scaled offset addressing modes (up to long but not quad)
 4371 operand immIScale()
 4372 %{
 4373   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4374   match(ConI);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 // 26 bit signed offset -- for pc-relative branches
 4382 operand immI26()
 4383 %{
 4384   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4385   match(ConI);
 4386 
 4387   op_cost(0);
 4388   format %{ %}
 4389   interface(CONST_INTER);
 4390 %}
 4391 
 4392 // 19 bit signed offset -- for pc-relative loads
 4393 operand immI19()
 4394 %{
 4395   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4396   match(ConI);
 4397 
 4398   op_cost(0);
 4399   format %{ %}
 4400   interface(CONST_INTER);
 4401 %}
 4402 
 4403 // 12 bit unsigned offset -- for base plus immediate loads
 4404 operand immIU12()
 4405 %{
 4406   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4407   match(ConI);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 operand immLU12()
 4415 %{
 4416   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4417   match(ConL);
 4418 
 4419   op_cost(0);
 4420   format %{ %}
 4421   interface(CONST_INTER);
 4422 %}
 4423 
 4424 // Offset for scaled or unscaled immediate loads and stores
 4425 operand immIOffset()
 4426 %{
 4427   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4428   match(ConI);
 4429 
 4430   op_cost(0);
 4431   format %{ %}
 4432   interface(CONST_INTER);
 4433 %}
 4434 
 4435 operand immIOffset1()
 4436 %{
 4437   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4438   match(ConI);
 4439 
 4440   op_cost(0);
 4441   format %{ %}
 4442   interface(CONST_INTER);
 4443 %}
 4444 
 4445 operand immIOffset2()
 4446 %{
 4447   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4448   match(ConI);
 4449 
 4450   op_cost(0);
 4451   format %{ %}
 4452   interface(CONST_INTER);
 4453 %}
 4454 
 4455 operand immIOffset4()
 4456 %{
 4457   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4458   match(ConI);
 4459 
 4460   op_cost(0);
 4461   format %{ %}
 4462   interface(CONST_INTER);
 4463 %}
 4464 
 4465 operand immIOffset8()
 4466 %{
 4467   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4468   match(ConI);
 4469 
 4470   op_cost(0);
 4471   format %{ %}
 4472   interface(CONST_INTER);
 4473 %}
 4474 
 4475 operand immIOffset16()
 4476 %{
 4477   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4478   match(ConI);
 4479 
 4480   op_cost(0);
 4481   format %{ %}
 4482   interface(CONST_INTER);
 4483 %}
 4484 
 4485 operand immLoffset()
 4486 %{
 4487   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4488   match(ConL);
 4489 
 4490   op_cost(0);
 4491   format %{ %}
 4492   interface(CONST_INTER);
 4493 %}
 4494 
 4495 operand immLoffset1()
 4496 %{
 4497   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4498   match(ConL);
 4499 
 4500   op_cost(0);
 4501   format %{ %}
 4502   interface(CONST_INTER);
 4503 %}
 4504 
 4505 operand immLoffset2()
 4506 %{
 4507   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4508   match(ConL);
 4509 
 4510   op_cost(0);
 4511   format %{ %}
 4512   interface(CONST_INTER);
 4513 %}
 4514 
 4515 operand immLoffset4()
 4516 %{
 4517   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4518   match(ConL);
 4519 
 4520   op_cost(0);
 4521   format %{ %}
 4522   interface(CONST_INTER);
 4523 %}
 4524 
 4525 operand immLoffset8()
 4526 %{
 4527   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4528   match(ConL);
 4529 
 4530   op_cost(0);
 4531   format %{ %}
 4532   interface(CONST_INTER);
 4533 %}
 4534 
 4535 operand immLoffset16()
 4536 %{
 4537   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4538   match(ConL);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 // 8 bit signed value.
 4546 operand immI8()
 4547 %{
 4548   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4549   match(ConI);
 4550 
 4551   op_cost(0);
 4552   format %{ %}
 4553   interface(CONST_INTER);
 4554 %}
 4555 
 4556 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4557 operand immI8_shift8()
 4558 %{
 4559   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4560             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4561   match(ConI);
 4562 
 4563   op_cost(0);
 4564   format %{ %}
 4565   interface(CONST_INTER);
 4566 %}
 4567 
 4568 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4569 operand immL8_shift8()
 4570 %{
 4571   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4572             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4573   match(ConL);
 4574 
 4575   op_cost(0);
 4576   format %{ %}
 4577   interface(CONST_INTER);
 4578 %}
 4579 
 4580 // 8 bit integer valid for vector add sub immediate
 4581 operand immBAddSubV()
 4582 %{
 4583   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4584   match(ConI);
 4585 
 4586   op_cost(0);
 4587   format %{ %}
 4588   interface(CONST_INTER);
 4589 %}
 4590 
 4591 // 32 bit integer valid for add sub immediate
 4592 operand immIAddSub()
 4593 %{
 4594   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4595   match(ConI);
 4596   op_cost(0);
 4597   format %{ %}
 4598   interface(CONST_INTER);
 4599 %}
 4600 
 4601 // 32 bit integer valid for vector add sub immediate
 4602 operand immIAddSubV()
 4603 %{
 4604   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4605   match(ConI);
 4606 
 4607   op_cost(0);
 4608   format %{ %}
 4609   interface(CONST_INTER);
 4610 %}
 4611 
 4612 // 32 bit unsigned integer valid for logical immediate
 4613 
 4614 operand immBLog()
 4615 %{
 4616   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4617   match(ConI);
 4618 
 4619   op_cost(0);
 4620   format %{ %}
 4621   interface(CONST_INTER);
 4622 %}
 4623 
 4624 operand immSLog()
 4625 %{
 4626   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4627   match(ConI);
 4628 
 4629   op_cost(0);
 4630   format %{ %}
 4631   interface(CONST_INTER);
 4632 %}
 4633 
 4634 operand immILog()
 4635 %{
 4636   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4637   match(ConI);
 4638 
 4639   op_cost(0);
 4640   format %{ %}
 4641   interface(CONST_INTER);
 4642 %}
 4643 
 4644 // Integer operands 64 bit
 4645 // 64 bit immediate
 4646 operand immL()
 4647 %{
 4648   match(ConL);
 4649 
 4650   op_cost(0);
 4651   format %{ %}
 4652   interface(CONST_INTER);
 4653 %}
 4654 
 4655 // 64 bit zero
 4656 operand immL0()
 4657 %{
 4658   predicate(n->get_long() == 0);
 4659   match(ConL);
 4660 
 4661   op_cost(0);
 4662   format %{ %}
 4663   interface(CONST_INTER);
 4664 %}
 4665 
 4666 // 64 bit unit increment
 4667 operand immL_1()
 4668 %{
 4669   predicate(n->get_long() == 1);
 4670   match(ConL);
 4671 
 4672   op_cost(0);
 4673   format %{ %}
 4674   interface(CONST_INTER);
 4675 %}
 4676 
 4677 // 64 bit unit decrement
 4678 operand immL_M1()
 4679 %{
 4680   predicate(n->get_long() == -1);
 4681   match(ConL);
 4682 
 4683   op_cost(0);
 4684   format %{ %}
 4685   interface(CONST_INTER);
 4686 %}
 4687 
 4688 // 32 bit offset of pc in thread anchor
 4689 
 4690 operand immL_pc_off()
 4691 %{
 4692   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4693                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4694   match(ConL);
 4695 
 4696   op_cost(0);
 4697   format %{ %}
 4698   interface(CONST_INTER);
 4699 %}
 4700 
 4701 // 64 bit integer valid for add sub immediate
 4702 operand immLAddSub()
 4703 %{
 4704   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4705   match(ConL);
 4706   op_cost(0);
 4707   format %{ %}
 4708   interface(CONST_INTER);
 4709 %}
 4710 
 4711 // 64 bit integer valid for addv subv immediate
 4712 operand immLAddSubV()
 4713 %{
 4714   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4715   match(ConL);
 4716 
 4717   op_cost(0);
 4718   format %{ %}
 4719   interface(CONST_INTER);
 4720 %}
 4721 
 4722 // 64 bit integer valid for logical immediate
 4723 operand immLLog()
 4724 %{
 4725   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4726   match(ConL);
 4727   op_cost(0);
 4728   format %{ %}
 4729   interface(CONST_INTER);
 4730 %}
 4731 
 4732 // Long Immediate: low 32-bit mask
 4733 operand immL_32bits()
 4734 %{
 4735   predicate(n->get_long() == 0xFFFFFFFFL);
 4736   match(ConL);
 4737   op_cost(0);
 4738   format %{ %}
 4739   interface(CONST_INTER);
 4740 %}
 4741 
 4742 // Pointer operands
 4743 // Pointer Immediate
 4744 operand immP()
 4745 %{
 4746   match(ConP);
 4747 
 4748   op_cost(0);
 4749   format %{ %}
 4750   interface(CONST_INTER);
 4751 %}
 4752 
 4753 // NULL Pointer Immediate
 4754 operand immP0()
 4755 %{
 4756   predicate(n->get_ptr() == 0);
 4757   match(ConP);
 4758 
 4759   op_cost(0);
 4760   format %{ %}
 4761   interface(CONST_INTER);
 4762 %}
 4763 
 4764 // Pointer Immediate One
 4765 // this is used in object initialization (initial object header)
 4766 operand immP_1()
 4767 %{
 4768   predicate(n->get_ptr() == 1);
 4769   match(ConP);
 4770 
 4771   op_cost(0);
 4772   format %{ %}
 4773   interface(CONST_INTER);
 4774 %}
 4775 
 4776 // Card Table Byte Map Base
 4777 operand immByteMapBase()
 4778 %{
 4779   // Get base of card map
 4780   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4781             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4782   match(ConP);
 4783 
 4784   op_cost(0);
 4785   format %{ %}
 4786   interface(CONST_INTER);
 4787 %}
 4788 
 4789 // Pointer Immediate Minus One
 4790 // this is used when we want to write the current PC to the thread anchor
 4791 operand immP_M1()
 4792 %{
 4793   predicate(n->get_ptr() == -1);
 4794   match(ConP);
 4795 
 4796   op_cost(0);
 4797   format %{ %}
 4798   interface(CONST_INTER);
 4799 %}
 4800 
 4801 // Pointer Immediate Minus Two
 4802 // this is used when we want to write the current PC to the thread anchor
 4803 operand immP_M2()
 4804 %{
 4805   predicate(n->get_ptr() == -2);
 4806   match(ConP);
 4807 
 4808   op_cost(0);
 4809   format %{ %}
 4810   interface(CONST_INTER);
 4811 %}
 4812 
 4813 // Float and Double operands
 4814 // Double Immediate
 4815 operand immD()
 4816 %{
 4817   match(ConD);
 4818   op_cost(0);
 4819   format %{ %}
 4820   interface(CONST_INTER);
 4821 %}
 4822 
 4823 // Double Immediate: +0.0d
 4824 operand immD0()
 4825 %{
 4826   predicate(jlong_cast(n->getd()) == 0);
 4827   match(ConD);
 4828 
 4829   op_cost(0);
 4830   format %{ %}
 4831   interface(CONST_INTER);
 4832 %}
 4833 
 4834 // constant 'double +0.0'.
 4835 operand immDPacked()
 4836 %{
 4837   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4838   match(ConD);
 4839   op_cost(0);
 4840   format %{ %}
 4841   interface(CONST_INTER);
 4842 %}
 4843 
 4844 // Float Immediate
 4845 operand immF()
 4846 %{
 4847   match(ConF);
 4848   op_cost(0);
 4849   format %{ %}
 4850   interface(CONST_INTER);
 4851 %}
 4852 
 4853 // Float Immediate: +0.0f.
 4854 operand immF0()
 4855 %{
 4856   predicate(jint_cast(n->getf()) == 0);
 4857   match(ConF);
 4858 
 4859   op_cost(0);
 4860   format %{ %}
 4861   interface(CONST_INTER);
 4862 %}
 4863 
 4864 //
 4865 operand immFPacked()
 4866 %{
 4867   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4868   match(ConF);
 4869   op_cost(0);
 4870   format %{ %}
 4871   interface(CONST_INTER);
 4872 %}
 4873 
 4874 // Narrow pointer operands
 4875 // Narrow Pointer Immediate
 4876 operand immN()
 4877 %{
 4878   match(ConN);
 4879 
 4880   op_cost(0);
 4881   format %{ %}
 4882   interface(CONST_INTER);
 4883 %}
 4884 
 4885 // Narrow NULL Pointer Immediate
 4886 operand immN0()
 4887 %{
 4888   predicate(n->get_narrowcon() == 0);
 4889   match(ConN);
 4890 
 4891   op_cost(0);
 4892   format %{ %}
 4893   interface(CONST_INTER);
 4894 %}
 4895 
 4896 operand immNKlass()
 4897 %{
 4898   match(ConNKlass);
 4899 
 4900   op_cost(0);
 4901   format %{ %}
 4902   interface(CONST_INTER);
 4903 %}
 4904 
 4905 // Integer 32 bit Register Operands
 4906 // Integer 32 bitRegister (excludes SP)
 4907 operand iRegI()
 4908 %{
 4909   constraint(ALLOC_IN_RC(any_reg32));
 4910   match(RegI);
 4911   match(iRegINoSp);
 4912   op_cost(0);
 4913   format %{ %}
 4914   interface(REG_INTER);
 4915 %}
 4916 
 4917 // Integer 32 bit Register not Special
 4918 operand iRegINoSp()
 4919 %{
 4920   constraint(ALLOC_IN_RC(no_special_reg32));
 4921   match(RegI);
 4922   op_cost(0);
 4923   format %{ %}
 4924   interface(REG_INTER);
 4925 %}
 4926 
 4927 // Integer 64 bit Register Operands
 4928 // Integer 64 bit Register (includes SP)
 4929 operand iRegL()
 4930 %{
 4931   constraint(ALLOC_IN_RC(any_reg));
 4932   match(RegL);
 4933   match(iRegLNoSp);
 4934   op_cost(0);
 4935   format %{ %}
 4936   interface(REG_INTER);
 4937 %}
 4938 
 4939 // Integer 64 bit Register not Special
 4940 operand iRegLNoSp()
 4941 %{
 4942   constraint(ALLOC_IN_RC(no_special_reg));
 4943   match(RegL);
 4944   match(iRegL_R0);
 4945   format %{ %}
 4946   interface(REG_INTER);
 4947 %}
 4948 
 4949 // Pointer Register Operands
 4950 // Pointer Register
 4951 operand iRegP()
 4952 %{
 4953   constraint(ALLOC_IN_RC(ptr_reg));
 4954   match(RegP);
 4955   match(iRegPNoSp);
 4956   match(iRegP_R0);
 4957   //match(iRegP_R2);
 4958   //match(iRegP_R4);
 4959   match(iRegP_R5);
 4960   match(thread_RegP);
 4961   op_cost(0);
 4962   format %{ %}
 4963   interface(REG_INTER);
 4964 %}
 4965 
 4966 // Pointer 64 bit Register not Special
 4967 operand iRegPNoSp()
 4968 %{
 4969   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4970   match(RegP);
 4971   // match(iRegP);
 4972   // match(iRegP_R0);
 4973   // match(iRegP_R2);
 4974   // match(iRegP_R4);
 4975   // match(iRegP_R5);
 4976   // match(thread_RegP);
 4977   op_cost(0);
 4978   format %{ %}
 4979   interface(REG_INTER);
 4980 %}
 4981 
 4982 // Pointer 64 bit Register R0 only
 4983 operand iRegP_R0()
 4984 %{
 4985   constraint(ALLOC_IN_RC(r0_reg));
 4986   match(RegP);
 4987   // match(iRegP);
 4988   match(iRegPNoSp);
 4989   op_cost(0);
 4990   format %{ %}
 4991   interface(REG_INTER);
 4992 %}
 4993 
 4994 // Pointer 64 bit Register R1 only
 4995 operand iRegP_R1()
 4996 %{
 4997   constraint(ALLOC_IN_RC(r1_reg));
 4998   match(RegP);
 4999   // match(iRegP);
 5000   match(iRegPNoSp);
 5001   op_cost(0);
 5002   format %{ %}
 5003   interface(REG_INTER);
 5004 %}
 5005 
 5006 // Pointer 64 bit Register R2 only
 5007 operand iRegP_R2()
 5008 %{
 5009   constraint(ALLOC_IN_RC(r2_reg));
 5010   match(RegP);
 5011   // match(iRegP);
 5012   match(iRegPNoSp);
 5013   op_cost(0);
 5014   format %{ %}
 5015   interface(REG_INTER);
 5016 %}
 5017 
 5018 // Pointer 64 bit Register R3 only
 5019 operand iRegP_R3()
 5020 %{
 5021   constraint(ALLOC_IN_RC(r3_reg));
 5022   match(RegP);
 5023   // match(iRegP);
 5024   match(iRegPNoSp);
 5025   op_cost(0);
 5026   format %{ %}
 5027   interface(REG_INTER);
 5028 %}
 5029 
 5030 // Pointer 64 bit Register R4 only
 5031 operand iRegP_R4()
 5032 %{
 5033   constraint(ALLOC_IN_RC(r4_reg));
 5034   match(RegP);
 5035   // match(iRegP);
 5036   match(iRegPNoSp);
 5037   op_cost(0);
 5038   format %{ %}
 5039   interface(REG_INTER);
 5040 %}
 5041 
 5042 // Pointer 64 bit Register R5 only
 5043 operand iRegP_R5()
 5044 %{
 5045   constraint(ALLOC_IN_RC(r5_reg));
 5046   match(RegP);
 5047   // match(iRegP);
 5048   match(iRegPNoSp);
 5049   op_cost(0);
 5050   format %{ %}
 5051   interface(REG_INTER);
 5052 %}
 5053 
 5054 // Pointer 64 bit Register R10 only
 5055 operand iRegP_R10()
 5056 %{
 5057   constraint(ALLOC_IN_RC(r10_reg));
 5058   match(RegP);
 5059   // match(iRegP);
 5060   match(iRegPNoSp);
 5061   op_cost(0);
 5062   format %{ %}
 5063   interface(REG_INTER);
 5064 %}
 5065 
 5066 // Long 64 bit Register R0 only
 5067 operand iRegL_R0()
 5068 %{
 5069   constraint(ALLOC_IN_RC(r0_reg));
 5070   match(RegL);
 5071   match(iRegLNoSp);
 5072   op_cost(0);
 5073   format %{ %}
 5074   interface(REG_INTER);
 5075 %}
 5076 
 5077 // Long 64 bit Register R2 only
 5078 operand iRegL_R2()
 5079 %{
 5080   constraint(ALLOC_IN_RC(r2_reg));
 5081   match(RegL);
 5082   match(iRegLNoSp);
 5083   op_cost(0);
 5084   format %{ %}
 5085   interface(REG_INTER);
 5086 %}
 5087 
 5088 // Long 64 bit Register R3 only
 5089 operand iRegL_R3()
 5090 %{
 5091   constraint(ALLOC_IN_RC(r3_reg));
 5092   match(RegL);
 5093   match(iRegLNoSp);
 5094   op_cost(0);
 5095   format %{ %}
 5096   interface(REG_INTER);
 5097 %}
 5098 
 5099 // Long 64 bit Register R11 only
 5100 operand iRegL_R11()
 5101 %{
 5102   constraint(ALLOC_IN_RC(r11_reg));
 5103   match(RegL);
 5104   match(iRegLNoSp);
 5105   op_cost(0);
 5106   format %{ %}
 5107   interface(REG_INTER);
 5108 %}
 5109 
 5110 // Pointer 64 bit Register FP only
 5111 operand iRegP_FP()
 5112 %{
 5113   constraint(ALLOC_IN_RC(fp_reg));
 5114   match(RegP);
 5115   // match(iRegP);
 5116   op_cost(0);
 5117   format %{ %}
 5118   interface(REG_INTER);
 5119 %}
 5120 
 5121 // Register R0 only
 5122 operand iRegI_R0()
 5123 %{
 5124   constraint(ALLOC_IN_RC(int_r0_reg));
 5125   match(RegI);
 5126   match(iRegINoSp);
 5127   op_cost(0);
 5128   format %{ %}
 5129   interface(REG_INTER);
 5130 %}
 5131 
 5132 // Register R2 only
 5133 operand iRegI_R2()
 5134 %{
 5135   constraint(ALLOC_IN_RC(int_r2_reg));
 5136   match(RegI);
 5137   match(iRegINoSp);
 5138   op_cost(0);
 5139   format %{ %}
 5140   interface(REG_INTER);
 5141 %}
 5142 
 5143 // Register R3 only
 5144 operand iRegI_R3()
 5145 %{
 5146   constraint(ALLOC_IN_RC(int_r3_reg));
 5147   match(RegI);
 5148   match(iRegINoSp);
 5149   op_cost(0);
 5150   format %{ %}
 5151   interface(REG_INTER);
 5152 %}
 5153 
 5154 
 5155 // Register R4 only
 5156 operand iRegI_R4()
 5157 %{
 5158   constraint(ALLOC_IN_RC(int_r4_reg));
 5159   match(RegI);
 5160   match(iRegINoSp);
 5161   op_cost(0);
 5162   format %{ %}
 5163   interface(REG_INTER);
 5164 %}
 5165 
 5166 
 5167 // Pointer Register Operands
 5168 // Narrow Pointer Register
 5169 operand iRegN()
 5170 %{
 5171   constraint(ALLOC_IN_RC(any_reg32));
 5172   match(RegN);
 5173   match(iRegNNoSp);
 5174   op_cost(0);
 5175   format %{ %}
 5176   interface(REG_INTER);
 5177 %}
 5178 
 5179 operand iRegN_R0()
 5180 %{
 5181   constraint(ALLOC_IN_RC(r0_reg));
 5182   match(iRegN);
 5183   op_cost(0);
 5184   format %{ %}
 5185   interface(REG_INTER);
 5186 %}
 5187 
 5188 operand iRegN_R2()
 5189 %{
 5190   constraint(ALLOC_IN_RC(r2_reg));
 5191   match(iRegN);
 5192   op_cost(0);
 5193   format %{ %}
 5194   interface(REG_INTER);
 5195 %}
 5196 
 5197 operand iRegN_R3()
 5198 %{
 5199   constraint(ALLOC_IN_RC(r3_reg));
 5200   match(iRegN);
 5201   op_cost(0);
 5202   format %{ %}
 5203   interface(REG_INTER);
 5204 %}
 5205 
 5206 // Integer 64 bit Register not Special
 5207 operand iRegNNoSp()
 5208 %{
 5209   constraint(ALLOC_IN_RC(no_special_reg32));
 5210   match(RegN);
 5211   op_cost(0);
 5212   format %{ %}
 5213   interface(REG_INTER);
 5214 %}
 5215 
 5216 // heap base register -- used for encoding immN0
 5217 
 5218 operand iRegIHeapbase()
 5219 %{
 5220   constraint(ALLOC_IN_RC(heapbase_reg));
 5221   match(RegI);
 5222   op_cost(0);
 5223   format %{ %}
 5224   interface(REG_INTER);
 5225 %}
 5226 
 5227 // Float Register
 5228 // Float register operands
 5229 operand vRegF()
 5230 %{
 5231   constraint(ALLOC_IN_RC(float_reg));
 5232   match(RegF);
 5233 
 5234   op_cost(0);
 5235   format %{ %}
 5236   interface(REG_INTER);
 5237 %}
 5238 
 5239 // Double Register
 5240 // Double register operands
 5241 operand vRegD()
 5242 %{
 5243   constraint(ALLOC_IN_RC(double_reg));
 5244   match(RegD);
 5245 
 5246   op_cost(0);
 5247   format %{ %}
 5248   interface(REG_INTER);
 5249 %}
 5250 
 5251 // Generic vector class. This will be used for
 5252 // all vector operands, including NEON and SVE.
 5253 operand vReg()
 5254 %{
 5255   constraint(ALLOC_IN_RC(dynamic));
 5256   match(VecA);
 5257   match(VecD);
 5258   match(VecX);
 5259 
 5260   op_cost(0);
 5261   format %{ %}
 5262   interface(REG_INTER);
 5263 %}
 5264 
 5265 operand vecA()
 5266 %{
 5267   constraint(ALLOC_IN_RC(vectora_reg));
 5268   match(VecA);
 5269 
 5270   op_cost(0);
 5271   format %{ %}
 5272   interface(REG_INTER);
 5273 %}
 5274 
 5275 operand vecD()
 5276 %{
 5277   constraint(ALLOC_IN_RC(vectord_reg));
 5278   match(VecD);
 5279 
 5280   op_cost(0);
 5281   format %{ %}
 5282   interface(REG_INTER);
 5283 %}
 5284 
 5285 operand vecX()
 5286 %{
 5287   constraint(ALLOC_IN_RC(vectorx_reg));
 5288   match(VecX);
 5289 
 5290   op_cost(0);
 5291   format %{ %}
 5292   interface(REG_INTER);
 5293 %}
 5294 
 5295 operand vRegD_V0()
 5296 %{
 5297   constraint(ALLOC_IN_RC(v0_reg));
 5298   match(RegD);
 5299   op_cost(0);
 5300   format %{ %}
 5301   interface(REG_INTER);
 5302 %}
 5303 
 5304 operand vRegD_V1()
 5305 %{
 5306   constraint(ALLOC_IN_RC(v1_reg));
 5307   match(RegD);
 5308   op_cost(0);
 5309   format %{ %}
 5310   interface(REG_INTER);
 5311 %}
 5312 
 5313 operand vRegD_V2()
 5314 %{
 5315   constraint(ALLOC_IN_RC(v2_reg));
 5316   match(RegD);
 5317   op_cost(0);
 5318   format %{ %}
 5319   interface(REG_INTER);
 5320 %}
 5321 
 5322 operand vRegD_V3()
 5323 %{
 5324   constraint(ALLOC_IN_RC(v3_reg));
 5325   match(RegD);
 5326   op_cost(0);
 5327   format %{ %}
 5328   interface(REG_INTER);
 5329 %}
 5330 
 5331 operand vRegD_V4()
 5332 %{
 5333   constraint(ALLOC_IN_RC(v4_reg));
 5334   match(RegD);
 5335   op_cost(0);
 5336   format %{ %}
 5337   interface(REG_INTER);
 5338 %}
 5339 
 5340 operand vRegD_V5()
 5341 %{
 5342   constraint(ALLOC_IN_RC(v5_reg));
 5343   match(RegD);
 5344   op_cost(0);
 5345   format %{ %}
 5346   interface(REG_INTER);
 5347 %}
 5348 
 5349 operand vRegD_V6()
 5350 %{
 5351   constraint(ALLOC_IN_RC(v6_reg));
 5352   match(RegD);
 5353   op_cost(0);
 5354   format %{ %}
 5355   interface(REG_INTER);
 5356 %}
 5357 
 5358 operand vRegD_V7()
 5359 %{
 5360   constraint(ALLOC_IN_RC(v7_reg));
 5361   match(RegD);
 5362   op_cost(0);
 5363   format %{ %}
 5364   interface(REG_INTER);
 5365 %}
 5366 
 5367 operand vRegD_V8()
 5368 %{
 5369   constraint(ALLOC_IN_RC(v8_reg));
 5370   match(RegD);
 5371   op_cost(0);
 5372   format %{ %}
 5373   interface(REG_INTER);
 5374 %}
 5375 
 5376 operand vRegD_V9()
 5377 %{
 5378   constraint(ALLOC_IN_RC(v9_reg));
 5379   match(RegD);
 5380   op_cost(0);
 5381   format %{ %}
 5382   interface(REG_INTER);
 5383 %}
 5384 
 5385 operand vRegD_V10()
 5386 %{
 5387   constraint(ALLOC_IN_RC(v10_reg));
 5388   match(RegD);
 5389   op_cost(0);
 5390   format %{ %}
 5391   interface(REG_INTER);
 5392 %}
 5393 
 5394 operand vRegD_V11()
 5395 %{
 5396   constraint(ALLOC_IN_RC(v11_reg));
 5397   match(RegD);
 5398   op_cost(0);
 5399   format %{ %}
 5400   interface(REG_INTER);
 5401 %}
 5402 
 5403 operand vRegD_V12()
 5404 %{
 5405   constraint(ALLOC_IN_RC(v12_reg));
 5406   match(RegD);
 5407   op_cost(0);
 5408   format %{ %}
 5409   interface(REG_INTER);
 5410 %}
 5411 
 5412 operand vRegD_V13()
 5413 %{
 5414   constraint(ALLOC_IN_RC(v13_reg));
 5415   match(RegD);
 5416   op_cost(0);
 5417   format %{ %}
 5418   interface(REG_INTER);
 5419 %}
 5420 
 5421 operand vRegD_V14()
 5422 %{
 5423   constraint(ALLOC_IN_RC(v14_reg));
 5424   match(RegD);
 5425   op_cost(0);
 5426   format %{ %}
 5427   interface(REG_INTER);
 5428 %}
 5429 
 5430 operand vRegD_V15()
 5431 %{
 5432   constraint(ALLOC_IN_RC(v15_reg));
 5433   match(RegD);
 5434   op_cost(0);
 5435   format %{ %}
 5436   interface(REG_INTER);
 5437 %}
 5438 
 5439 operand vRegD_V16()
 5440 %{
 5441   constraint(ALLOC_IN_RC(v16_reg));
 5442   match(RegD);
 5443   op_cost(0);
 5444   format %{ %}
 5445   interface(REG_INTER);
 5446 %}
 5447 
 5448 operand vRegD_V17()
 5449 %{
 5450   constraint(ALLOC_IN_RC(v17_reg));
 5451   match(RegD);
 5452   op_cost(0);
 5453   format %{ %}
 5454   interface(REG_INTER);
 5455 %}
 5456 
 5457 operand vRegD_V18()
 5458 %{
 5459   constraint(ALLOC_IN_RC(v18_reg));
 5460   match(RegD);
 5461   op_cost(0);
 5462   format %{ %}
 5463   interface(REG_INTER);
 5464 %}
 5465 
 5466 operand vRegD_V19()
 5467 %{
 5468   constraint(ALLOC_IN_RC(v19_reg));
 5469   match(RegD);
 5470   op_cost(0);
 5471   format %{ %}
 5472   interface(REG_INTER);
 5473 %}
 5474 
 5475 operand vRegD_V20()
 5476 %{
 5477   constraint(ALLOC_IN_RC(v20_reg));
 5478   match(RegD);
 5479   op_cost(0);
 5480   format %{ %}
 5481   interface(REG_INTER);
 5482 %}
 5483 
 5484 operand vRegD_V21()
 5485 %{
 5486   constraint(ALLOC_IN_RC(v21_reg));
 5487   match(RegD);
 5488   op_cost(0);
 5489   format %{ %}
 5490   interface(REG_INTER);
 5491 %}
 5492 
 5493 operand vRegD_V22()
 5494 %{
 5495   constraint(ALLOC_IN_RC(v22_reg));
 5496   match(RegD);
 5497   op_cost(0);
 5498   format %{ %}
 5499   interface(REG_INTER);
 5500 %}
 5501 
 5502 operand vRegD_V23()
 5503 %{
 5504   constraint(ALLOC_IN_RC(v23_reg));
 5505   match(RegD);
 5506   op_cost(0);
 5507   format %{ %}
 5508   interface(REG_INTER);
 5509 %}
 5510 
 5511 operand vRegD_V24()
 5512 %{
 5513   constraint(ALLOC_IN_RC(v24_reg));
 5514   match(RegD);
 5515   op_cost(0);
 5516   format %{ %}
 5517   interface(REG_INTER);
 5518 %}
 5519 
 5520 operand vRegD_V25()
 5521 %{
 5522   constraint(ALLOC_IN_RC(v25_reg));
 5523   match(RegD);
 5524   op_cost(0);
 5525   format %{ %}
 5526   interface(REG_INTER);
 5527 %}
 5528 
 5529 operand vRegD_V26()
 5530 %{
 5531   constraint(ALLOC_IN_RC(v26_reg));
 5532   match(RegD);
 5533   op_cost(0);
 5534   format %{ %}
 5535   interface(REG_INTER);
 5536 %}
 5537 
 5538 operand vRegD_V27()
 5539 %{
 5540   constraint(ALLOC_IN_RC(v27_reg));
 5541   match(RegD);
 5542   op_cost(0);
 5543   format %{ %}
 5544   interface(REG_INTER);
 5545 %}
 5546 
 5547 operand vRegD_V28()
 5548 %{
 5549   constraint(ALLOC_IN_RC(v28_reg));
 5550   match(RegD);
 5551   op_cost(0);
 5552   format %{ %}
 5553   interface(REG_INTER);
 5554 %}
 5555 
 5556 operand vRegD_V29()
 5557 %{
 5558   constraint(ALLOC_IN_RC(v29_reg));
 5559   match(RegD);
 5560   op_cost(0);
 5561   format %{ %}
 5562   interface(REG_INTER);
 5563 %}
 5564 
 5565 operand vRegD_V30()
 5566 %{
 5567   constraint(ALLOC_IN_RC(v30_reg));
 5568   match(RegD);
 5569   op_cost(0);
 5570   format %{ %}
 5571   interface(REG_INTER);
 5572 %}
 5573 
 5574 operand vRegD_V31()
 5575 %{
 5576   constraint(ALLOC_IN_RC(v31_reg));
 5577   match(RegD);
 5578   op_cost(0);
 5579   format %{ %}
 5580   interface(REG_INTER);
 5581 %}
 5582 
 5583 operand pReg()
 5584 %{
 5585   constraint(ALLOC_IN_RC(pr_reg));
 5586   match(RegVectMask);
 5587   match(pRegGov);
 5588   op_cost(0);
 5589   format %{ %}
 5590   interface(REG_INTER);
 5591 %}
 5592 
 5593 operand pRegGov()
 5594 %{
 5595   constraint(ALLOC_IN_RC(gov_pr));
 5596   match(RegVectMask);
 5597   match(pReg);
 5598   op_cost(0);
 5599   format %{ %}
 5600   interface(REG_INTER);
 5601 %}
 5602 
 5603 operand pRegGov_P0()
 5604 %{
 5605   constraint(ALLOC_IN_RC(p0_reg));
 5606   match(RegVectMask);
 5607   op_cost(0);
 5608   format %{ %}
 5609   interface(REG_INTER);
 5610 %}
 5611 
 5612 operand pRegGov_P1()
 5613 %{
 5614   constraint(ALLOC_IN_RC(p1_reg));
 5615   match(RegVectMask);
 5616   op_cost(0);
 5617   format %{ %}
 5618   interface(REG_INTER);
 5619 %}
 5620 
 5621 // Flags register, used as output of signed compare instructions
 5622 
 5623 // note that on AArch64 we also use this register as the output for
 5624 // for floating point compare instructions (CmpF CmpD). this ensures
 5625 // that ordered inequality tests use GT, GE, LT or LE none of which
 5626 // pass through cases where the result is unordered i.e. one or both
 5627 // inputs to the compare is a NaN. this means that the ideal code can
 5628 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5629 // (where the comparison should always fail). EQ and NE tests are
 5630 // always generated in ideal code so that unordered folds into the NE
 5631 // case, matching the behaviour of AArch64 NE.
 5632 //
 5633 // This differs from x86 where the outputs of FP compares use a
 5634 // special FP flags registers and where compares based on this
 5635 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5636 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5637 // to explicitly handle the unordered case in branches. x86 also has
 5638 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5639 
 5640 operand rFlagsReg()
 5641 %{
 5642   constraint(ALLOC_IN_RC(int_flags));
 5643   match(RegFlags);
 5644 
 5645   op_cost(0);
 5646   format %{ "RFLAGS" %}
 5647   interface(REG_INTER);
 5648 %}
 5649 
 5650 // Flags register, used as output of unsigned compare instructions
 5651 operand rFlagsRegU()
 5652 %{
 5653   constraint(ALLOC_IN_RC(int_flags));
 5654   match(RegFlags);
 5655 
 5656   op_cost(0);
 5657   format %{ "RFLAGSU" %}
 5658   interface(REG_INTER);
 5659 %}
 5660 
 5661 // Special Registers
 5662 
 5663 // Method Register
 5664 operand inline_cache_RegP(iRegP reg)
 5665 %{
 5666   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5667   match(reg);
 5668   match(iRegPNoSp);
 5669   op_cost(0);
 5670   format %{ %}
 5671   interface(REG_INTER);
 5672 %}
 5673 
 5674 // Thread Register
 5675 operand thread_RegP(iRegP reg)
 5676 %{
 5677   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5678   match(reg);
 5679   op_cost(0);
 5680   format %{ %}
 5681   interface(REG_INTER);
 5682 %}
 5683 
 5684 operand lr_RegP(iRegP reg)
 5685 %{
 5686   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5687   match(reg);
 5688   op_cost(0);
 5689   format %{ %}
 5690   interface(REG_INTER);
 5691 %}
 5692 
 5693 //----------Memory Operands----------------------------------------------------
 5694 
 5695 operand indirect(iRegP reg)
 5696 %{
 5697   constraint(ALLOC_IN_RC(ptr_reg));
 5698   match(reg);
 5699   op_cost(0);
 5700   format %{ "[$reg]" %}
 5701   interface(MEMORY_INTER) %{
 5702     base($reg);
 5703     index(0xffffffff);
 5704     scale(0x0);
 5705     disp(0x0);
 5706   %}
 5707 %}
 5708 
 5709 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5710 %{
 5711   constraint(ALLOC_IN_RC(ptr_reg));
 5712   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5713   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5714   op_cost(0);
 5715   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5716   interface(MEMORY_INTER) %{
 5717     base($reg);
 5718     index($ireg);
 5719     scale($scale);
 5720     disp(0x0);
 5721   %}
 5722 %}
 5723 
 5724 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5725 %{
 5726   constraint(ALLOC_IN_RC(ptr_reg));
 5727   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5728   match(AddP reg (LShiftL lreg scale));
 5729   op_cost(0);
 5730   format %{ "$reg, $lreg lsl($scale)" %}
 5731   interface(MEMORY_INTER) %{
 5732     base($reg);
 5733     index($lreg);
 5734     scale($scale);
 5735     disp(0x0);
 5736   %}
 5737 %}
 5738 
 5739 operand indIndexI2L(iRegP reg, iRegI ireg)
 5740 %{
 5741   constraint(ALLOC_IN_RC(ptr_reg));
 5742   match(AddP reg (ConvI2L ireg));
 5743   op_cost(0);
 5744   format %{ "$reg, $ireg, 0, I2L" %}
 5745   interface(MEMORY_INTER) %{
 5746     base($reg);
 5747     index($ireg);
 5748     scale(0x0);
 5749     disp(0x0);
 5750   %}
 5751 %}
 5752 
 5753 operand indIndex(iRegP reg, iRegL lreg)
 5754 %{
 5755   constraint(ALLOC_IN_RC(ptr_reg));
 5756   match(AddP reg lreg);
 5757   op_cost(0);
 5758   format %{ "$reg, $lreg" %}
 5759   interface(MEMORY_INTER) %{
 5760     base($reg);
 5761     index($lreg);
 5762     scale(0x0);
 5763     disp(0x0);
 5764   %}
 5765 %}
 5766 
 5767 operand indOffI(iRegP reg, immIOffset off)
 5768 %{
 5769   constraint(ALLOC_IN_RC(ptr_reg));
 5770   match(AddP reg off);
 5771   op_cost(0);
 5772   format %{ "[$reg, $off]" %}
 5773   interface(MEMORY_INTER) %{
 5774     base($reg);
 5775     index(0xffffffff);
 5776     scale(0x0);
 5777     disp($off);
 5778   %}
 5779 %}
 5780 
 5781 operand indOffI1(iRegP reg, immIOffset1 off)
 5782 %{
 5783   constraint(ALLOC_IN_RC(ptr_reg));
 5784   match(AddP reg off);
 5785   op_cost(0);
 5786   format %{ "[$reg, $off]" %}
 5787   interface(MEMORY_INTER) %{
 5788     base($reg);
 5789     index(0xffffffff);
 5790     scale(0x0);
 5791     disp($off);
 5792   %}
 5793 %}
 5794 
 5795 operand indOffI2(iRegP reg, immIOffset2 off)
 5796 %{
 5797   constraint(ALLOC_IN_RC(ptr_reg));
 5798   match(AddP reg off);
 5799   op_cost(0);
 5800   format %{ "[$reg, $off]" %}
 5801   interface(MEMORY_INTER) %{
 5802     base($reg);
 5803     index(0xffffffff);
 5804     scale(0x0);
 5805     disp($off);
 5806   %}
 5807 %}
 5808 
 5809 operand indOffI4(iRegP reg, immIOffset4 off)
 5810 %{
 5811   constraint(ALLOC_IN_RC(ptr_reg));
 5812   match(AddP reg off);
 5813   op_cost(0);
 5814   format %{ "[$reg, $off]" %}
 5815   interface(MEMORY_INTER) %{
 5816     base($reg);
 5817     index(0xffffffff);
 5818     scale(0x0);
 5819     disp($off);
 5820   %}
 5821 %}
 5822 
 5823 operand indOffI8(iRegP reg, immIOffset8 off)
 5824 %{
 5825   constraint(ALLOC_IN_RC(ptr_reg));
 5826   match(AddP reg off);
 5827   op_cost(0);
 5828   format %{ "[$reg, $off]" %}
 5829   interface(MEMORY_INTER) %{
 5830     base($reg);
 5831     index(0xffffffff);
 5832     scale(0x0);
 5833     disp($off);
 5834   %}
 5835 %}
 5836 
 5837 operand indOffI16(iRegP reg, immIOffset16 off)
 5838 %{
 5839   constraint(ALLOC_IN_RC(ptr_reg));
 5840   match(AddP reg off);
 5841   op_cost(0);
 5842   format %{ "[$reg, $off]" %}
 5843   interface(MEMORY_INTER) %{
 5844     base($reg);
 5845     index(0xffffffff);
 5846     scale(0x0);
 5847     disp($off);
 5848   %}
 5849 %}
 5850 
 5851 operand indOffL(iRegP reg, immLoffset off)
 5852 %{
 5853   constraint(ALLOC_IN_RC(ptr_reg));
 5854   match(AddP reg off);
 5855   op_cost(0);
 5856   format %{ "[$reg, $off]" %}
 5857   interface(MEMORY_INTER) %{
 5858     base($reg);
 5859     index(0xffffffff);
 5860     scale(0x0);
 5861     disp($off);
 5862   %}
 5863 %}
 5864 
 5865 operand indOffL1(iRegP reg, immLoffset1 off)
 5866 %{
 5867   constraint(ALLOC_IN_RC(ptr_reg));
 5868   match(AddP reg off);
 5869   op_cost(0);
 5870   format %{ "[$reg, $off]" %}
 5871   interface(MEMORY_INTER) %{
 5872     base($reg);
 5873     index(0xffffffff);
 5874     scale(0x0);
 5875     disp($off);
 5876   %}
 5877 %}
 5878 
 5879 operand indOffL2(iRegP reg, immLoffset2 off)
 5880 %{
 5881   constraint(ALLOC_IN_RC(ptr_reg));
 5882   match(AddP reg off);
 5883   op_cost(0);
 5884   format %{ "[$reg, $off]" %}
 5885   interface(MEMORY_INTER) %{
 5886     base($reg);
 5887     index(0xffffffff);
 5888     scale(0x0);
 5889     disp($off);
 5890   %}
 5891 %}
 5892 
 5893 operand indOffL4(iRegP reg, immLoffset4 off)
 5894 %{
 5895   constraint(ALLOC_IN_RC(ptr_reg));
 5896   match(AddP reg off);
 5897   op_cost(0);
 5898   format %{ "[$reg, $off]" %}
 5899   interface(MEMORY_INTER) %{
 5900     base($reg);
 5901     index(0xffffffff);
 5902     scale(0x0);
 5903     disp($off);
 5904   %}
 5905 %}
 5906 
 5907 operand indOffL8(iRegP reg, immLoffset8 off)
 5908 %{
 5909   constraint(ALLOC_IN_RC(ptr_reg));
 5910   match(AddP reg off);
 5911   op_cost(0);
 5912   format %{ "[$reg, $off]" %}
 5913   interface(MEMORY_INTER) %{
 5914     base($reg);
 5915     index(0xffffffff);
 5916     scale(0x0);
 5917     disp($off);
 5918   %}
 5919 %}
 5920 
 5921 operand indOffL16(iRegP reg, immLoffset16 off)
 5922 %{
 5923   constraint(ALLOC_IN_RC(ptr_reg));
 5924   match(AddP reg off);
 5925   op_cost(0);
 5926   format %{ "[$reg, $off]" %}
 5927   interface(MEMORY_INTER) %{
 5928     base($reg);
 5929     index(0xffffffff);
 5930     scale(0x0);
 5931     disp($off);
 5932   %}
 5933 %}
 5934 
 5935 operand indirectN(iRegN reg)
 5936 %{
 5937   predicate(CompressedOops::shift() == 0);
 5938   constraint(ALLOC_IN_RC(ptr_reg));
 5939   match(DecodeN reg);
 5940   op_cost(0);
 5941   format %{ "[$reg]\t# narrow" %}
 5942   interface(MEMORY_INTER) %{
 5943     base($reg);
 5944     index(0xffffffff);
 5945     scale(0x0);
 5946     disp(0x0);
 5947   %}
 5948 %}
 5949 
 5950 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5951 %{
 5952   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5953   constraint(ALLOC_IN_RC(ptr_reg));
 5954   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5955   op_cost(0);
 5956   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5957   interface(MEMORY_INTER) %{
 5958     base($reg);
 5959     index($ireg);
 5960     scale($scale);
 5961     disp(0x0);
 5962   %}
 5963 %}
 5964 
 5965 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5966 %{
 5967   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5968   constraint(ALLOC_IN_RC(ptr_reg));
 5969   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5970   op_cost(0);
 5971   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5972   interface(MEMORY_INTER) %{
 5973     base($reg);
 5974     index($lreg);
 5975     scale($scale);
 5976     disp(0x0);
 5977   %}
 5978 %}
 5979 
 5980 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5981 %{
 5982   predicate(CompressedOops::shift() == 0);
 5983   constraint(ALLOC_IN_RC(ptr_reg));
 5984   match(AddP (DecodeN reg) (ConvI2L ireg));
 5985   op_cost(0);
 5986   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5987   interface(MEMORY_INTER) %{
 5988     base($reg);
 5989     index($ireg);
 5990     scale(0x0);
 5991     disp(0x0);
 5992   %}
 5993 %}
 5994 
 5995 operand indIndexN(iRegN reg, iRegL lreg)
 5996 %{
 5997   predicate(CompressedOops::shift() == 0);
 5998   constraint(ALLOC_IN_RC(ptr_reg));
 5999   match(AddP (DecodeN reg) lreg);
 6000   op_cost(0);
 6001   format %{ "$reg, $lreg\t# narrow" %}
 6002   interface(MEMORY_INTER) %{
 6003     base($reg);
 6004     index($lreg);
 6005     scale(0x0);
 6006     disp(0x0);
 6007   %}
 6008 %}
 6009 
 6010 operand indOffIN(iRegN reg, immIOffset off)
 6011 %{
 6012   predicate(CompressedOops::shift() == 0);
 6013   constraint(ALLOC_IN_RC(ptr_reg));
 6014   match(AddP (DecodeN reg) off);
 6015   op_cost(0);
 6016   format %{ "[$reg, $off]\t# narrow" %}
 6017   interface(MEMORY_INTER) %{
 6018     base($reg);
 6019     index(0xffffffff);
 6020     scale(0x0);
 6021     disp($off);
 6022   %}
 6023 %}
 6024 
 6025 operand indOffLN(iRegN reg, immLoffset off)
 6026 %{
 6027   predicate(CompressedOops::shift() == 0);
 6028   constraint(ALLOC_IN_RC(ptr_reg));
 6029   match(AddP (DecodeN reg) off);
 6030   op_cost(0);
 6031   format %{ "[$reg, $off]\t# narrow" %}
 6032   interface(MEMORY_INTER) %{
 6033     base($reg);
 6034     index(0xffffffff);
 6035     scale(0x0);
 6036     disp($off);
 6037   %}
 6038 %}
 6039 
 6040 
 6041 
 6042 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6043 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6044 %{
 6045   constraint(ALLOC_IN_RC(ptr_reg));
 6046   match(AddP reg off);
 6047   op_cost(0);
 6048   format %{ "[$reg, $off]" %}
 6049   interface(MEMORY_INTER) %{
 6050     base($reg);
 6051     index(0xffffffff);
 6052     scale(0x0);
 6053     disp($off);
 6054   %}
 6055 %}
 6056 
 6057 //----------Special Memory Operands--------------------------------------------
 6058 // Stack Slot Operand - This operand is used for loading and storing temporary
 6059 //                      values on the stack where a match requires a value to
 6060 //                      flow through memory.
 6061 operand stackSlotP(sRegP reg)
 6062 %{
 6063   constraint(ALLOC_IN_RC(stack_slots));
 6064   op_cost(100);
 6065   // No match rule because this operand is only generated in matching
 6066   // match(RegP);
 6067   format %{ "[$reg]" %}
 6068   interface(MEMORY_INTER) %{
 6069     base(0x1e);  // RSP
 6070     index(0x0);  // No Index
 6071     scale(0x0);  // No Scale
 6072     disp($reg);  // Stack Offset
 6073   %}
 6074 %}
 6075 
 6076 operand stackSlotI(sRegI reg)
 6077 %{
 6078   constraint(ALLOC_IN_RC(stack_slots));
 6079   // No match rule because this operand is only generated in matching
 6080   // match(RegI);
 6081   format %{ "[$reg]" %}
 6082   interface(MEMORY_INTER) %{
 6083     base(0x1e);  // RSP
 6084     index(0x0);  // No Index
 6085     scale(0x0);  // No Scale
 6086     disp($reg);  // Stack Offset
 6087   %}
 6088 %}
 6089 
 6090 operand stackSlotF(sRegF reg)
 6091 %{
 6092   constraint(ALLOC_IN_RC(stack_slots));
 6093   // No match rule because this operand is only generated in matching
 6094   // match(RegF);
 6095   format %{ "[$reg]" %}
 6096   interface(MEMORY_INTER) %{
 6097     base(0x1e);  // RSP
 6098     index(0x0);  // No Index
 6099     scale(0x0);  // No Scale
 6100     disp($reg);  // Stack Offset
 6101   %}
 6102 %}
 6103 
 6104 operand stackSlotD(sRegD reg)
 6105 %{
 6106   constraint(ALLOC_IN_RC(stack_slots));
 6107   // No match rule because this operand is only generated in matching
 6108   // match(RegD);
 6109   format %{ "[$reg]" %}
 6110   interface(MEMORY_INTER) %{
 6111     base(0x1e);  // RSP
 6112     index(0x0);  // No Index
 6113     scale(0x0);  // No Scale
 6114     disp($reg);  // Stack Offset
 6115   %}
 6116 %}
 6117 
 6118 operand stackSlotL(sRegL reg)
 6119 %{
 6120   constraint(ALLOC_IN_RC(stack_slots));
 6121   // No match rule because this operand is only generated in matching
 6122   // match(RegL);
 6123   format %{ "[$reg]" %}
 6124   interface(MEMORY_INTER) %{
 6125     base(0x1e);  // RSP
 6126     index(0x0);  // No Index
 6127     scale(0x0);  // No Scale
 6128     disp($reg);  // Stack Offset
 6129   %}
 6130 %}
 6131 
 6132 // Operands for expressing Control Flow
 6133 // NOTE: Label is a predefined operand which should not be redefined in
 6134 //       the AD file. It is generically handled within the ADLC.
 6135 
 6136 //----------Conditional Branch Operands----------------------------------------
 6137 // Comparison Op  - This is the operation of the comparison, and is limited to
 6138 //                  the following set of codes:
 6139 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6140 //
 6141 // Other attributes of the comparison, such as unsignedness, are specified
 6142 // by the comparison instruction that sets a condition code flags register.
 6143 // That result is represented by a flags operand whose subtype is appropriate
 6144 // to the unsignedness (etc.) of the comparison.
 6145 //
 6146 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6147 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6148 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6149 
 6150 // used for signed integral comparisons and fp comparisons
 6151 
 6152 operand cmpOp()
 6153 %{
 6154   match(Bool);
 6155 
 6156   format %{ "" %}
 6157   interface(COND_INTER) %{
 6158     equal(0x0, "eq");
 6159     not_equal(0x1, "ne");
 6160     less(0xb, "lt");
 6161     greater_equal(0xa, "ge");
 6162     less_equal(0xd, "le");
 6163     greater(0xc, "gt");
 6164     overflow(0x6, "vs");
 6165     no_overflow(0x7, "vc");
 6166   %}
 6167 %}
 6168 
 6169 // used for unsigned integral comparisons
 6170 
 6171 operand cmpOpU()
 6172 %{
 6173   match(Bool);
 6174 
 6175   format %{ "" %}
 6176   interface(COND_INTER) %{
 6177     equal(0x0, "eq");
 6178     not_equal(0x1, "ne");
 6179     less(0x3, "lo");
 6180     greater_equal(0x2, "hs");
 6181     less_equal(0x9, "ls");
 6182     greater(0x8, "hi");
 6183     overflow(0x6, "vs");
 6184     no_overflow(0x7, "vc");
 6185   %}
 6186 %}
 6187 
 6188 // used for certain integral comparisons which can be
 6189 // converted to cbxx or tbxx instructions
 6190 
 6191 operand cmpOpEqNe()
 6192 %{
 6193   match(Bool);
 6194   op_cost(0);
 6195   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6196             || n->as_Bool()->_test._test == BoolTest::eq);
 6197 
 6198   format %{ "" %}
 6199   interface(COND_INTER) %{
 6200     equal(0x0, "eq");
 6201     not_equal(0x1, "ne");
 6202     less(0xb, "lt");
 6203     greater_equal(0xa, "ge");
 6204     less_equal(0xd, "le");
 6205     greater(0xc, "gt");
 6206     overflow(0x6, "vs");
 6207     no_overflow(0x7, "vc");
 6208   %}
 6209 %}
 6210 
 6211 // used for certain integral comparisons which can be
 6212 // converted to cbxx or tbxx instructions
 6213 
 6214 operand cmpOpLtGe()
 6215 %{
 6216   match(Bool);
 6217   op_cost(0);
 6218 
 6219   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6220             || n->as_Bool()->_test._test == BoolTest::ge);
 6221 
 6222   format %{ "" %}
 6223   interface(COND_INTER) %{
 6224     equal(0x0, "eq");
 6225     not_equal(0x1, "ne");
 6226     less(0xb, "lt");
 6227     greater_equal(0xa, "ge");
 6228     less_equal(0xd, "le");
 6229     greater(0xc, "gt");
 6230     overflow(0x6, "vs");
 6231     no_overflow(0x7, "vc");
 6232   %}
 6233 %}
 6234 
 6235 // used for certain unsigned integral comparisons which can be
 6236 // converted to cbxx or tbxx instructions
 6237 
 6238 operand cmpOpUEqNeLtGe()
 6239 %{
 6240   match(Bool);
 6241   op_cost(0);
 6242 
 6243   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6244             || n->as_Bool()->_test._test == BoolTest::ne
 6245             || n->as_Bool()->_test._test == BoolTest::lt
 6246             || n->as_Bool()->_test._test == BoolTest::ge);
 6247 
 6248   format %{ "" %}
 6249   interface(COND_INTER) %{
 6250     equal(0x0, "eq");
 6251     not_equal(0x1, "ne");
 6252     less(0xb, "lt");
 6253     greater_equal(0xa, "ge");
 6254     less_equal(0xd, "le");
 6255     greater(0xc, "gt");
 6256     overflow(0x6, "vs");
 6257     no_overflow(0x7, "vc");
 6258   %}
 6259 %}
 6260 
 6261 // Special operand allowing long args to int ops to be truncated for free
 6262 
 6263 operand iRegL2I(iRegL reg) %{
 6264 
 6265   op_cost(0);
 6266 
 6267   match(ConvL2I reg);
 6268 
 6269   format %{ "l2i($reg)" %}
 6270 
 6271   interface(REG_INTER)
 6272 %}
 6273 
 6274 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6275 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6276 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6277 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6278 
 6279 //----------OPERAND CLASSES----------------------------------------------------
 6280 // Operand Classes are groups of operands that are used as to simplify
 6281 // instruction definitions by not requiring the AD writer to specify
 6282 // separate instructions for every form of operand when the
 6283 // instruction accepts multiple operand types with the same basic
 6284 // encoding and format. The classic case of this is memory operands.
 6285 
 6286 // memory is used to define read/write location for load/store
 6287 // instruction defs. we can turn a memory op into an Address
 6288 
 6289 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6290                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6291 
 6292 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6293                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6294 
 6295 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6296                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6297 
 6298 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6299                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6300 
 6301 // All of the memory operands. For the pipeline description.
 6302 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6303                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6304                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6305 
 6306 
 6307 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6308 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6309 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6310 // can be elided because the 32-bit instruction will just employ the
 6311 // lower 32 bits anyway.
 6312 //
 6313 // n.b. this does not elide all L2I conversions. if the truncated
 6314 // value is consumed by more than one operation then the ConvL2I
 6315 // cannot be bundled into the consuming nodes so an l2i gets planted
 6316 // (actually a movw $dst $src) and the downstream instructions consume
 6317 // the result of the l2i as an iRegI input. That's a shame since the
 6318 // movw is actually redundant but its not too costly.
 6319 
 6320 opclass iRegIorL2I(iRegI, iRegL2I);
 6321 
 6322 //----------PIPELINE-----------------------------------------------------------
 6323 // Rules which define the behavior of the target architectures pipeline.
 6324 
 6325 // For specific pipelines, eg A53, define the stages of that pipeline
 6326 //pipe_desc(ISS, EX1, EX2, WR);
 6327 #define ISS S0
 6328 #define EX1 S1
 6329 #define EX2 S2
 6330 #define WR  S3
 6331 
 6332 // Integer ALU reg operation
 6333 pipeline %{
 6334 
 6335 attributes %{
 6336   // ARM instructions are of fixed length
 6337   fixed_size_instructions;        // Fixed size instructions TODO does
 6338   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6339   // ARM instructions come in 32-bit word units
 6340   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6341   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6342   instruction_fetch_units = 1;       // of 64 bytes
 6343 
 6344   // List of nop instructions
 6345   nops( MachNop );
 6346 %}
 6347 
 6348 // We don't use an actual pipeline model so don't care about resources
 6349 // or description. we do use pipeline classes to introduce fixed
 6350 // latencies
 6351 
 6352 //----------RESOURCES----------------------------------------------------------
 6353 // Resources are the functional units available to the machine
 6354 
 6355 resources( INS0, INS1, INS01 = INS0 | INS1,
 6356            ALU0, ALU1, ALU = ALU0 | ALU1,
 6357            MAC,
 6358            DIV,
 6359            BRANCH,
 6360            LDST,
 6361            NEON_FP);
 6362 
 6363 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6364 // Pipeline Description specifies the stages in the machine's pipeline
 6365 
 6366 // Define the pipeline as a generic 6 stage pipeline
 6367 pipe_desc(S0, S1, S2, S3, S4, S5);
 6368 
 6369 //----------PIPELINE CLASSES---------------------------------------------------
 6370 // Pipeline Classes describe the stages in which input and output are
 6371 // referenced by the hardware pipeline.
 6372 
 6373 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6374 %{
 6375   single_instruction;
 6376   src1   : S1(read);
 6377   src2   : S2(read);
 6378   dst    : S5(write);
 6379   INS01  : ISS;
 6380   NEON_FP : S5;
 6381 %}
 6382 
 6383 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6384 %{
 6385   single_instruction;
 6386   src1   : S1(read);
 6387   src2   : S2(read);
 6388   dst    : S5(write);
 6389   INS01  : ISS;
 6390   NEON_FP : S5;
 6391 %}
 6392 
 6393 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6394 %{
 6395   single_instruction;
 6396   src    : S1(read);
 6397   dst    : S5(write);
 6398   INS01  : ISS;
 6399   NEON_FP : S5;
 6400 %}
 6401 
 6402 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6403 %{
 6404   single_instruction;
 6405   src    : S1(read);
 6406   dst    : S5(write);
 6407   INS01  : ISS;
 6408   NEON_FP : S5;
 6409 %}
 6410 
 6411 pipe_class fp_d2f(vRegF dst, vRegD src)
 6412 %{
 6413   single_instruction;
 6414   src    : S1(read);
 6415   dst    : S5(write);
 6416   INS01  : ISS;
 6417   NEON_FP : S5;
 6418 %}
 6419 
 6420 pipe_class fp_f2d(vRegD dst, vRegF src)
 6421 %{
 6422   single_instruction;
 6423   src    : S1(read);
 6424   dst    : S5(write);
 6425   INS01  : ISS;
 6426   NEON_FP : S5;
 6427 %}
 6428 
 6429 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6430 %{
 6431   single_instruction;
 6432   src    : S1(read);
 6433   dst    : S5(write);
 6434   INS01  : ISS;
 6435   NEON_FP : S5;
 6436 %}
 6437 
 6438 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6439 %{
 6440   single_instruction;
 6441   src    : S1(read);
 6442   dst    : S5(write);
 6443   INS01  : ISS;
 6444   NEON_FP : S5;
 6445 %}
 6446 
 6447 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6448 %{
 6449   single_instruction;
 6450   src    : S1(read);
 6451   dst    : S5(write);
 6452   INS01  : ISS;
 6453   NEON_FP : S5;
 6454 %}
 6455 
 6456 pipe_class fp_l2f(vRegF dst, iRegL src)
 6457 %{
 6458   single_instruction;
 6459   src    : S1(read);
 6460   dst    : S5(write);
 6461   INS01  : ISS;
 6462   NEON_FP : S5;
 6463 %}
 6464 
 6465 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6466 %{
 6467   single_instruction;
 6468   src    : S1(read);
 6469   dst    : S5(write);
 6470   INS01  : ISS;
 6471   NEON_FP : S5;
 6472 %}
 6473 
 6474 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6475 %{
 6476   single_instruction;
 6477   src    : S1(read);
 6478   dst    : S5(write);
 6479   INS01  : ISS;
 6480   NEON_FP : S5;
 6481 %}
 6482 
 6483 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6484 %{
 6485   single_instruction;
 6486   src    : S1(read);
 6487   dst    : S5(write);
 6488   INS01  : ISS;
 6489   NEON_FP : S5;
 6490 %}
 6491 
 6492 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6493 %{
 6494   single_instruction;
 6495   src    : S1(read);
 6496   dst    : S5(write);
 6497   INS01  : ISS;
 6498   NEON_FP : S5;
 6499 %}
 6500 
 6501 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6502 %{
 6503   single_instruction;
 6504   src1   : S1(read);
 6505   src2   : S2(read);
 6506   dst    : S5(write);
 6507   INS0   : ISS;
 6508   NEON_FP : S5;
 6509 %}
 6510 
 6511 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6512 %{
 6513   single_instruction;
 6514   src1   : S1(read);
 6515   src2   : S2(read);
 6516   dst    : S5(write);
 6517   INS0   : ISS;
 6518   NEON_FP : S5;
 6519 %}
 6520 
 6521 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6522 %{
 6523   single_instruction;
 6524   cr     : S1(read);
 6525   src1   : S1(read);
 6526   src2   : S1(read);
 6527   dst    : S3(write);
 6528   INS01  : ISS;
 6529   NEON_FP : S3;
 6530 %}
 6531 
 6532 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6533 %{
 6534   single_instruction;
 6535   cr     : S1(read);
 6536   src1   : S1(read);
 6537   src2   : S1(read);
 6538   dst    : S3(write);
 6539   INS01  : ISS;
 6540   NEON_FP : S3;
 6541 %}
 6542 
 6543 pipe_class fp_imm_s(vRegF dst)
 6544 %{
 6545   single_instruction;
 6546   dst    : S3(write);
 6547   INS01  : ISS;
 6548   NEON_FP : S3;
 6549 %}
 6550 
 6551 pipe_class fp_imm_d(vRegD dst)
 6552 %{
 6553   single_instruction;
 6554   dst    : S3(write);
 6555   INS01  : ISS;
 6556   NEON_FP : S3;
 6557 %}
 6558 
 6559 pipe_class fp_load_constant_s(vRegF dst)
 6560 %{
 6561   single_instruction;
 6562   dst    : S4(write);
 6563   INS01  : ISS;
 6564   NEON_FP : S4;
 6565 %}
 6566 
 6567 pipe_class fp_load_constant_d(vRegD dst)
 6568 %{
 6569   single_instruction;
 6570   dst    : S4(write);
 6571   INS01  : ISS;
 6572   NEON_FP : S4;
 6573 %}
 6574 
 6575 //------- Integer ALU operations --------------------------
 6576 
 6577 // Integer ALU reg-reg operation
 6578 // Operands needed in EX1, result generated in EX2
 6579 // Eg.  ADD     x0, x1, x2
 6580 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6581 %{
 6582   single_instruction;
 6583   dst    : EX2(write);
 6584   src1   : EX1(read);
 6585   src2   : EX1(read);
 6586   INS01  : ISS; // Dual issue as instruction 0 or 1
 6587   ALU    : EX2;
 6588 %}
 6589 
 6590 // Integer ALU reg-reg operation with constant shift
 6591 // Shifted register must be available in LATE_ISS instead of EX1
 6592 // Eg.  ADD     x0, x1, x2, LSL #2
 6593 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6594 %{
 6595   single_instruction;
 6596   dst    : EX2(write);
 6597   src1   : EX1(read);
 6598   src2   : ISS(read);
 6599   INS01  : ISS;
 6600   ALU    : EX2;
 6601 %}
 6602 
 6603 // Integer ALU reg operation with constant shift
 6604 // Eg.  LSL     x0, x1, #shift
 6605 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6606 %{
 6607   single_instruction;
 6608   dst    : EX2(write);
 6609   src1   : ISS(read);
 6610   INS01  : ISS;
 6611   ALU    : EX2;
 6612 %}
 6613 
 6614 // Integer ALU reg-reg operation with variable shift
 6615 // Both operands must be available in LATE_ISS instead of EX1
 6616 // Result is available in EX1 instead of EX2
 6617 // Eg.  LSLV    x0, x1, x2
 6618 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6619 %{
 6620   single_instruction;
 6621   dst    : EX1(write);
 6622   src1   : ISS(read);
 6623   src2   : ISS(read);
 6624   INS01  : ISS;
 6625   ALU    : EX1;
 6626 %}
 6627 
 6628 // Integer ALU reg-reg operation with extract
 6629 // As for _vshift above, but result generated in EX2
 6630 // Eg.  EXTR    x0, x1, x2, #N
 6631 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6632 %{
 6633   single_instruction;
 6634   dst    : EX2(write);
 6635   src1   : ISS(read);
 6636   src2   : ISS(read);
 6637   INS1   : ISS; // Can only dual issue as Instruction 1
 6638   ALU    : EX1;
 6639 %}
 6640 
 6641 // Integer ALU reg operation
 6642 // Eg.  NEG     x0, x1
 6643 pipe_class ialu_reg(iRegI dst, iRegI src)
 6644 %{
 6645   single_instruction;
 6646   dst    : EX2(write);
 6647   src    : EX1(read);
 6648   INS01  : ISS;
 6649   ALU    : EX2;
 6650 %}
 6651 
 6652 // Integer ALU reg mmediate operation
 6653 // Eg.  ADD     x0, x1, #N
 6654 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6655 %{
 6656   single_instruction;
 6657   dst    : EX2(write);
 6658   src1   : EX1(read);
 6659   INS01  : ISS;
 6660   ALU    : EX2;
 6661 %}
 6662 
 6663 // Integer ALU immediate operation (no source operands)
 6664 // Eg.  MOV     x0, #N
 6665 pipe_class ialu_imm(iRegI dst)
 6666 %{
 6667   single_instruction;
 6668   dst    : EX1(write);
 6669   INS01  : ISS;
 6670   ALU    : EX1;
 6671 %}
 6672 
 6673 //------- Compare operation -------------------------------
 6674 
 6675 // Compare reg-reg
 6676 // Eg.  CMP     x0, x1
 6677 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6678 %{
 6679   single_instruction;
 6680 //  fixed_latency(16);
 6681   cr     : EX2(write);
 6682   op1    : EX1(read);
 6683   op2    : EX1(read);
 6684   INS01  : ISS;
 6685   ALU    : EX2;
 6686 %}
 6687 
 6688 // Compare reg-reg
 6689 // Eg.  CMP     x0, #N
 6690 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6691 %{
 6692   single_instruction;
 6693 //  fixed_latency(16);
 6694   cr     : EX2(write);
 6695   op1    : EX1(read);
 6696   INS01  : ISS;
 6697   ALU    : EX2;
 6698 %}
 6699 
 6700 //------- Conditional instructions ------------------------
 6701 
 6702 // Conditional no operands
 6703 // Eg.  CSINC   x0, zr, zr, <cond>
 6704 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6705 %{
 6706   single_instruction;
 6707   cr     : EX1(read);
 6708   dst    : EX2(write);
 6709   INS01  : ISS;
 6710   ALU    : EX2;
 6711 %}
 6712 
 6713 // Conditional 2 operand
 6714 // EG.  CSEL    X0, X1, X2, <cond>
 6715 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6716 %{
 6717   single_instruction;
 6718   cr     : EX1(read);
 6719   src1   : EX1(read);
 6720   src2   : EX1(read);
 6721   dst    : EX2(write);
 6722   INS01  : ISS;
 6723   ALU    : EX2;
 6724 %}
 6725 
 6726 // Conditional 2 operand
 6727 // EG.  CSEL    X0, X1, X2, <cond>
 6728 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6729 %{
 6730   single_instruction;
 6731   cr     : EX1(read);
 6732   src    : EX1(read);
 6733   dst    : EX2(write);
 6734   INS01  : ISS;
 6735   ALU    : EX2;
 6736 %}
 6737 
 6738 //------- Multiply pipeline operations --------------------
 6739 
 6740 // Multiply reg-reg
 6741 // Eg.  MUL     w0, w1, w2
 6742 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6743 %{
 6744   single_instruction;
 6745   dst    : WR(write);
 6746   src1   : ISS(read);
 6747   src2   : ISS(read);
 6748   INS01  : ISS;
 6749   MAC    : WR;
 6750 %}
 6751 
 6752 // Multiply accumulate
 6753 // Eg.  MADD    w0, w1, w2, w3
 6754 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6755 %{
 6756   single_instruction;
 6757   dst    : WR(write);
 6758   src1   : ISS(read);
 6759   src2   : ISS(read);
 6760   src3   : ISS(read);
 6761   INS01  : ISS;
 6762   MAC    : WR;
 6763 %}
 6764 
 6765 // Eg.  MUL     w0, w1, w2
 6766 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6767 %{
 6768   single_instruction;
 6769   fixed_latency(3); // Maximum latency for 64 bit mul
 6770   dst    : WR(write);
 6771   src1   : ISS(read);
 6772   src2   : ISS(read);
 6773   INS01  : ISS;
 6774   MAC    : WR;
 6775 %}
 6776 
 6777 // Multiply accumulate
 6778 // Eg.  MADD    w0, w1, w2, w3
 6779 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6780 %{
 6781   single_instruction;
 6782   fixed_latency(3); // Maximum latency for 64 bit mul
 6783   dst    : WR(write);
 6784   src1   : ISS(read);
 6785   src2   : ISS(read);
 6786   src3   : ISS(read);
 6787   INS01  : ISS;
 6788   MAC    : WR;
 6789 %}
 6790 
 6791 //------- Divide pipeline operations --------------------
 6792 
 6793 // Eg.  SDIV    w0, w1, w2
 6794 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6795 %{
 6796   single_instruction;
 6797   fixed_latency(8); // Maximum latency for 32 bit divide
 6798   dst    : WR(write);
 6799   src1   : ISS(read);
 6800   src2   : ISS(read);
 6801   INS0   : ISS; // Can only dual issue as instruction 0
 6802   DIV    : WR;
 6803 %}
 6804 
 6805 // Eg.  SDIV    x0, x1, x2
 6806 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6807 %{
 6808   single_instruction;
 6809   fixed_latency(16); // Maximum latency for 64 bit divide
 6810   dst    : WR(write);
 6811   src1   : ISS(read);
 6812   src2   : ISS(read);
 6813   INS0   : ISS; // Can only dual issue as instruction 0
 6814   DIV    : WR;
 6815 %}
 6816 
 6817 //------- Load pipeline operations ------------------------
 6818 
 6819 // Load - prefetch
 6820 // Eg.  PFRM    <mem>
 6821 pipe_class iload_prefetch(memory mem)
 6822 %{
 6823   single_instruction;
 6824   mem    : ISS(read);
 6825   INS01  : ISS;
 6826   LDST   : WR;
 6827 %}
 6828 
 6829 // Load - reg, mem
 6830 // Eg.  LDR     x0, <mem>
 6831 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6832 %{
 6833   single_instruction;
 6834   dst    : WR(write);
 6835   mem    : ISS(read);
 6836   INS01  : ISS;
 6837   LDST   : WR;
 6838 %}
 6839 
 6840 // Load - reg, reg
 6841 // Eg.  LDR     x0, [sp, x1]
 6842 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6843 %{
 6844   single_instruction;
 6845   dst    : WR(write);
 6846   src    : ISS(read);
 6847   INS01  : ISS;
 6848   LDST   : WR;
 6849 %}
 6850 
 6851 //------- Store pipeline operations -----------------------
 6852 
 6853 // Store - zr, mem
 6854 // Eg.  STR     zr, <mem>
 6855 pipe_class istore_mem(memory mem)
 6856 %{
 6857   single_instruction;
 6858   mem    : ISS(read);
 6859   INS01  : ISS;
 6860   LDST   : WR;
 6861 %}
 6862 
 6863 // Store - reg, mem
 6864 // Eg.  STR     x0, <mem>
 6865 pipe_class istore_reg_mem(iRegI src, memory mem)
 6866 %{
 6867   single_instruction;
 6868   mem    : ISS(read);
 6869   src    : EX2(read);
 6870   INS01  : ISS;
 6871   LDST   : WR;
 6872 %}
 6873 
 6874 // Store - reg, reg
 6875 // Eg. STR      x0, [sp, x1]
 6876 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6877 %{
 6878   single_instruction;
 6879   dst    : ISS(read);
 6880   src    : EX2(read);
 6881   INS01  : ISS;
 6882   LDST   : WR;
 6883 %}
 6884 
 6885 //------- Store pipeline operations -----------------------
 6886 
 6887 // Branch
 6888 pipe_class pipe_branch()
 6889 %{
 6890   single_instruction;
 6891   INS01  : ISS;
 6892   BRANCH : EX1;
 6893 %}
 6894 
 6895 // Conditional branch
 6896 pipe_class pipe_branch_cond(rFlagsReg cr)
 6897 %{
 6898   single_instruction;
 6899   cr     : EX1(read);
 6900   INS01  : ISS;
 6901   BRANCH : EX1;
 6902 %}
 6903 
 6904 // Compare & Branch
 6905 // EG.  CBZ/CBNZ
 6906 pipe_class pipe_cmp_branch(iRegI op1)
 6907 %{
 6908   single_instruction;
 6909   op1    : EX1(read);
 6910   INS01  : ISS;
 6911   BRANCH : EX1;
 6912 %}
 6913 
 6914 //------- Synchronisation operations ----------------------
 6915 
 6916 // Any operation requiring serialization.
 6917 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6918 pipe_class pipe_serial()
 6919 %{
 6920   single_instruction;
 6921   force_serialization;
 6922   fixed_latency(16);
 6923   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6924   LDST   : WR;
 6925 %}
 6926 
 6927 // Generic big/slow expanded idiom - also serialized
 6928 pipe_class pipe_slow()
 6929 %{
 6930   instruction_count(10);
 6931   multiple_bundles;
 6932   force_serialization;
 6933   fixed_latency(16);
 6934   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6935   LDST   : WR;
 6936 %}
 6937 
 6938 // Empty pipeline class
 6939 pipe_class pipe_class_empty()
 6940 %{
 6941   single_instruction;
 6942   fixed_latency(0);
 6943 %}
 6944 
 6945 // Default pipeline class.
 6946 pipe_class pipe_class_default()
 6947 %{
 6948   single_instruction;
 6949   fixed_latency(2);
 6950 %}
 6951 
 6952 // Pipeline class for compares.
 6953 pipe_class pipe_class_compare()
 6954 %{
 6955   single_instruction;
 6956   fixed_latency(16);
 6957 %}
 6958 
 6959 // Pipeline class for memory operations.
 6960 pipe_class pipe_class_memory()
 6961 %{
 6962   single_instruction;
 6963   fixed_latency(16);
 6964 %}
 6965 
 6966 // Pipeline class for call.
 6967 pipe_class pipe_class_call()
 6968 %{
 6969   single_instruction;
 6970   fixed_latency(100);
 6971 %}
 6972 
 6973 // Define the class for the Nop node.
 6974 define %{
 6975    MachNop = pipe_class_empty;
 6976 %}
 6977 
 6978 %}
 6979 //----------INSTRUCTIONS-------------------------------------------------------
 6980 //
 6981 // match      -- States which machine-independent subtree may be replaced
 6982 //               by this instruction.
 6983 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6984 //               selection to identify a minimum cost tree of machine
 6985 //               instructions that matches a tree of machine-independent
 6986 //               instructions.
 6987 // format     -- A string providing the disassembly for this instruction.
 6988 //               The value of an instruction's operand may be inserted
 6989 //               by referring to it with a '$' prefix.
 6990 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6991 //               to within an encode class as $primary, $secondary, and $tertiary
 6992 //               rrspectively.  The primary opcode is commonly used to
 6993 //               indicate the type of machine instruction, while secondary
 6994 //               and tertiary are often used for prefix options or addressing
 6995 //               modes.
 6996 // ins_encode -- A list of encode classes with parameters. The encode class
 6997 //               name must have been defined in an 'enc_class' specification
 6998 //               in the encode section of the architecture description.
 6999 
 7000 // ============================================================================
 7001 // Memory (Load/Store) Instructions
 7002 
 7003 // Load Instructions
 7004 
 7005 // Load Byte (8 bit signed)
 7006 instruct loadB(iRegINoSp dst, memory1 mem)
 7007 %{
 7008   match(Set dst (LoadB mem));
 7009   predicate(!needs_acquiring_load(n));
 7010 
 7011   ins_cost(4 * INSN_COST);
 7012   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7013 
 7014   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7015 
 7016   ins_pipe(iload_reg_mem);
 7017 %}
 7018 
 7019 // Load Byte (8 bit signed) into long
 7020 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7021 %{
 7022   match(Set dst (ConvI2L (LoadB mem)));
 7023   predicate(!needs_acquiring_load(n->in(1)));
 7024 
 7025   ins_cost(4 * INSN_COST);
 7026   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7027 
 7028   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7029 
 7030   ins_pipe(iload_reg_mem);
 7031 %}
 7032 
 7033 // Load Byte (8 bit unsigned)
 7034 instruct loadUB(iRegINoSp dst, memory1 mem)
 7035 %{
 7036   match(Set dst (LoadUB mem));
 7037   predicate(!needs_acquiring_load(n));
 7038 
 7039   ins_cost(4 * INSN_COST);
 7040   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7041 
 7042   ins_encode(aarch64_enc_ldrb(dst, mem));
 7043 
 7044   ins_pipe(iload_reg_mem);
 7045 %}
 7046 
 7047 // Load Byte (8 bit unsigned) into long
 7048 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7049 %{
 7050   match(Set dst (ConvI2L (LoadUB mem)));
 7051   predicate(!needs_acquiring_load(n->in(1)));
 7052 
 7053   ins_cost(4 * INSN_COST);
 7054   format %{ "ldrb  $dst, $mem\t# byte" %}
 7055 
 7056   ins_encode(aarch64_enc_ldrb(dst, mem));
 7057 
 7058   ins_pipe(iload_reg_mem);
 7059 %}
 7060 
 7061 // Load Short (16 bit signed)
 7062 instruct loadS(iRegINoSp dst, memory2 mem)
 7063 %{
 7064   match(Set dst (LoadS mem));
 7065   predicate(!needs_acquiring_load(n));
 7066 
 7067   ins_cost(4 * INSN_COST);
 7068   format %{ "ldrshw  $dst, $mem\t# short" %}
 7069 
 7070   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7071 
 7072   ins_pipe(iload_reg_mem);
 7073 %}
 7074 
 7075 // Load Short (16 bit signed) into long
 7076 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7077 %{
 7078   match(Set dst (ConvI2L (LoadS mem)));
 7079   predicate(!needs_acquiring_load(n->in(1)));
 7080 
 7081   ins_cost(4 * INSN_COST);
 7082   format %{ "ldrsh  $dst, $mem\t# short" %}
 7083 
 7084   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7085 
 7086   ins_pipe(iload_reg_mem);
 7087 %}
 7088 
 7089 // Load Char (16 bit unsigned)
 7090 instruct loadUS(iRegINoSp dst, memory2 mem)
 7091 %{
 7092   match(Set dst (LoadUS mem));
 7093   predicate(!needs_acquiring_load(n));
 7094 
 7095   ins_cost(4 * INSN_COST);
 7096   format %{ "ldrh  $dst, $mem\t# short" %}
 7097 
 7098   ins_encode(aarch64_enc_ldrh(dst, mem));
 7099 
 7100   ins_pipe(iload_reg_mem);
 7101 %}
 7102 
 7103 // Load Short/Char (16 bit unsigned) into long
 7104 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7105 %{
 7106   match(Set dst (ConvI2L (LoadUS mem)));
 7107   predicate(!needs_acquiring_load(n->in(1)));
 7108 
 7109   ins_cost(4 * INSN_COST);
 7110   format %{ "ldrh  $dst, $mem\t# short" %}
 7111 
 7112   ins_encode(aarch64_enc_ldrh(dst, mem));
 7113 
 7114   ins_pipe(iload_reg_mem);
 7115 %}
 7116 
 7117 // Load Integer (32 bit signed)
 7118 instruct loadI(iRegINoSp dst, memory4 mem)
 7119 %{
 7120   match(Set dst (LoadI mem));
 7121   predicate(!needs_acquiring_load(n));
 7122 
 7123   ins_cost(4 * INSN_COST);
 7124   format %{ "ldrw  $dst, $mem\t# int" %}
 7125 
 7126   ins_encode(aarch64_enc_ldrw(dst, mem));
 7127 
 7128   ins_pipe(iload_reg_mem);
 7129 %}
 7130 
 7131 // Load Integer (32 bit signed) into long
 7132 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7133 %{
 7134   match(Set dst (ConvI2L (LoadI mem)));
 7135   predicate(!needs_acquiring_load(n->in(1)));
 7136 
 7137   ins_cost(4 * INSN_COST);
 7138   format %{ "ldrsw  $dst, $mem\t# int" %}
 7139 
 7140   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7141 
 7142   ins_pipe(iload_reg_mem);
 7143 %}
 7144 
 7145 // Load Integer (32 bit unsigned) into long
 7146 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7147 %{
 7148   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7149   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7150 
 7151   ins_cost(4 * INSN_COST);
 7152   format %{ "ldrw  $dst, $mem\t# int" %}
 7153 
 7154   ins_encode(aarch64_enc_ldrw(dst, mem));
 7155 
 7156   ins_pipe(iload_reg_mem);
 7157 %}
 7158 
 7159 // Load Long (64 bit signed)
 7160 instruct loadL(iRegLNoSp dst, memory8 mem)
 7161 %{
 7162   match(Set dst (LoadL mem));
 7163   predicate(!needs_acquiring_load(n));
 7164 
 7165   ins_cost(4 * INSN_COST);
 7166   format %{ "ldr  $dst, $mem\t# int" %}
 7167 
 7168   ins_encode(aarch64_enc_ldr(dst, mem));
 7169 
 7170   ins_pipe(iload_reg_mem);
 7171 %}
 7172 
 7173 // Load Range
 7174 instruct loadRange(iRegINoSp dst, memory4 mem)
 7175 %{
 7176   match(Set dst (LoadRange mem));
 7177 
 7178   ins_cost(4 * INSN_COST);
 7179   format %{ "ldrw  $dst, $mem\t# range" %}
 7180 
 7181   ins_encode(aarch64_enc_ldrw(dst, mem));
 7182 
 7183   ins_pipe(iload_reg_mem);
 7184 %}
 7185 
 7186 // Load Pointer
 7187 instruct loadP(iRegPNoSp dst, memory8 mem)
 7188 %{
 7189   match(Set dst (LoadP mem));
 7190   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7191 
 7192   ins_cost(4 * INSN_COST);
 7193   format %{ "ldr  $dst, $mem\t# ptr" %}
 7194 
 7195   ins_encode(aarch64_enc_ldr(dst, mem));
 7196 
 7197   ins_pipe(iload_reg_mem);
 7198 %}
 7199 
 7200 // Load Compressed Pointer
 7201 instruct loadN(iRegNNoSp dst, memory4 mem)
 7202 %{
 7203   match(Set dst (LoadN mem));
 7204   predicate(!needs_acquiring_load(n));
 7205 
 7206   ins_cost(4 * INSN_COST);
 7207   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7208 
 7209   ins_encode(aarch64_enc_ldrw(dst, mem));
 7210 
 7211   ins_pipe(iload_reg_mem);
 7212 %}
 7213 
 7214 // Load Klass Pointer
 7215 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7216 %{
 7217   match(Set dst (LoadKlass mem));
 7218   predicate(!needs_acquiring_load(n));
 7219 
 7220   ins_cost(4 * INSN_COST);
 7221   format %{ "ldr  $dst, $mem\t# class" %}
 7222 
 7223   ins_encode(aarch64_enc_ldr(dst, mem));
 7224 
 7225   ins_pipe(iload_reg_mem);
 7226 %}
 7227 
 7228 // Load Narrow Klass Pointer
 7229 instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
 7230 %{
 7231   match(Set dst (LoadNKlass mem));
 7232   effect(TEMP_DEF dst, KILL cr);
 7233   predicate(!needs_acquiring_load(n));
 7234 
 7235   ins_cost(4 * INSN_COST);
 7236   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7237   ins_encode %{
 7238     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
 7239     assert($mem$$index$$Register == noreg, "expect no index");
 7240     Register dst = $dst$$Register;
 7241     Register obj = $mem$$base$$Register;
 7242     C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
 7243     Compile::current()->output()->add_stub(stub);
 7244     __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
 7245     // NOTE: We can't use tbnz here, because the target is sometimes too far away
 7246     // and cannot be encoded.
 7247     __ tst(dst, markWord::monitor_value);
 7248     __ br(Assembler::NE, stub->entry());
 7249     __ bind(stub->continuation());
 7250     __ lsr(dst, dst, markWord::klass_shift);
 7251   %}
 7252   ins_pipe(pipe_slow);
 7253 %}
 7254 
 7255 // Load Float
 7256 instruct loadF(vRegF dst, memory4 mem)
 7257 %{
 7258   match(Set dst (LoadF mem));
 7259   predicate(!needs_acquiring_load(n));
 7260 
 7261   ins_cost(4 * INSN_COST);
 7262   format %{ "ldrs  $dst, $mem\t# float" %}
 7263 
 7264   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7265 
 7266   ins_pipe(pipe_class_memory);
 7267 %}
 7268 
 7269 // Load Double
 7270 instruct loadD(vRegD dst, memory8 mem)
 7271 %{
 7272   match(Set dst (LoadD mem));
 7273   predicate(!needs_acquiring_load(n));
 7274 
 7275   ins_cost(4 * INSN_COST);
 7276   format %{ "ldrd  $dst, $mem\t# double" %}
 7277 
 7278   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7279 
 7280   ins_pipe(pipe_class_memory);
 7281 %}
 7282 
 7283 
 7284 // Load Int Constant
 7285 instruct loadConI(iRegINoSp dst, immI src)
 7286 %{
 7287   match(Set dst src);
 7288 
 7289   ins_cost(INSN_COST);
 7290   format %{ "mov $dst, $src\t# int" %}
 7291 
 7292   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7293 
 7294   ins_pipe(ialu_imm);
 7295 %}
 7296 
 7297 // Load Long Constant
 7298 instruct loadConL(iRegLNoSp dst, immL src)
 7299 %{
 7300   match(Set dst src);
 7301 
 7302   ins_cost(INSN_COST);
 7303   format %{ "mov $dst, $src\t# long" %}
 7304 
 7305   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7306 
 7307   ins_pipe(ialu_imm);
 7308 %}
 7309 
 7310 // Load Pointer Constant
 7311 
 7312 instruct loadConP(iRegPNoSp dst, immP con)
 7313 %{
 7314   match(Set dst con);
 7315 
 7316   ins_cost(INSN_COST * 4);
 7317   format %{
 7318     "mov  $dst, $con\t# ptr\n\t"
 7319   %}
 7320 
 7321   ins_encode(aarch64_enc_mov_p(dst, con));
 7322 
 7323   ins_pipe(ialu_imm);
 7324 %}
 7325 
 7326 // Load Null Pointer Constant
 7327 
 7328 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7329 %{
 7330   match(Set dst con);
 7331 
 7332   ins_cost(INSN_COST);
 7333   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7334 
 7335   ins_encode(aarch64_enc_mov_p0(dst, con));
 7336 
 7337   ins_pipe(ialu_imm);
 7338 %}
 7339 
 7340 // Load Pointer Constant One
 7341 
 7342 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7343 %{
 7344   match(Set dst con);
 7345 
 7346   ins_cost(INSN_COST);
 7347   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7348 
 7349   ins_encode(aarch64_enc_mov_p1(dst, con));
 7350 
 7351   ins_pipe(ialu_imm);
 7352 %}
 7353 
 7354 // Load Byte Map Base Constant
 7355 
 7356 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7357 %{
 7358   match(Set dst con);
 7359 
 7360   ins_cost(INSN_COST);
 7361   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7362 
 7363   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7364 
 7365   ins_pipe(ialu_imm);
 7366 %}
 7367 
 7368 // Load Narrow Pointer Constant
 7369 
 7370 instruct loadConN(iRegNNoSp dst, immN con)
 7371 %{
 7372   match(Set dst con);
 7373 
 7374   ins_cost(INSN_COST * 4);
 7375   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7376 
 7377   ins_encode(aarch64_enc_mov_n(dst, con));
 7378 
 7379   ins_pipe(ialu_imm);
 7380 %}
 7381 
 7382 // Load Narrow Null Pointer Constant
 7383 
 7384 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7385 %{
 7386   match(Set dst con);
 7387 
 7388   ins_cost(INSN_COST);
 7389   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7390 
 7391   ins_encode(aarch64_enc_mov_n0(dst, con));
 7392 
 7393   ins_pipe(ialu_imm);
 7394 %}
 7395 
 7396 // Load Narrow Klass Constant
 7397 
 7398 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7399 %{
 7400   match(Set dst con);
 7401 
 7402   ins_cost(INSN_COST);
 7403   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7404 
 7405   ins_encode(aarch64_enc_mov_nk(dst, con));
 7406 
 7407   ins_pipe(ialu_imm);
 7408 %}
 7409 
 7410 // Load Packed Float Constant
 7411 
 7412 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7413   match(Set dst con);
 7414   ins_cost(INSN_COST * 4);
 7415   format %{ "fmovs  $dst, $con"%}
 7416   ins_encode %{
 7417     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7418   %}
 7419 
 7420   ins_pipe(fp_imm_s);
 7421 %}
 7422 
 7423 // Load Float Constant
 7424 
 7425 instruct loadConF(vRegF dst, immF con) %{
 7426   match(Set dst con);
 7427 
 7428   ins_cost(INSN_COST * 4);
 7429 
 7430   format %{
 7431     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7432   %}
 7433 
 7434   ins_encode %{
 7435     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7436   %}
 7437 
 7438   ins_pipe(fp_load_constant_s);
 7439 %}
 7440 
 7441 // Load Packed Double Constant
 7442 
 7443 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7444   match(Set dst con);
 7445   ins_cost(INSN_COST);
 7446   format %{ "fmovd  $dst, $con"%}
 7447   ins_encode %{
 7448     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7449   %}
 7450 
 7451   ins_pipe(fp_imm_d);
 7452 %}
 7453 
 7454 // Load Double Constant
 7455 
 7456 instruct loadConD(vRegD dst, immD con) %{
 7457   match(Set dst con);
 7458 
 7459   ins_cost(INSN_COST * 5);
 7460   format %{
 7461     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7462   %}
 7463 
 7464   ins_encode %{
 7465     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7466   %}
 7467 
 7468   ins_pipe(fp_load_constant_d);
 7469 %}
 7470 
 7471 // Store Instructions
 7472 
 7473 // Store CMS card-mark Immediate
 7474 instruct storeimmCM0(immI0 zero, memory1 mem)
 7475 %{
 7476   match(Set mem (StoreCM mem zero));
 7477 
 7478   ins_cost(INSN_COST);
 7479   format %{ "storestore (elided)\n\t"
 7480             "strb zr, $mem\t# byte" %}
 7481 
 7482   ins_encode(aarch64_enc_strb0(mem));
 7483 
 7484   ins_pipe(istore_mem);
 7485 %}
 7486 
 7487 // Store CMS card-mark Immediate with intervening StoreStore
 7488 // needed when using CMS with no conditional card marking
 7489 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7490 %{
 7491   match(Set mem (StoreCM mem zero));
 7492 
 7493   ins_cost(INSN_COST * 2);
 7494   format %{ "storestore\n\t"
 7495             "dmb ishst"
 7496             "\n\tstrb zr, $mem\t# byte" %}
 7497 
 7498   ins_encode(aarch64_enc_strb0_ordered(mem));
 7499 
 7500   ins_pipe(istore_mem);
 7501 %}
 7502 
 7503 // Store Byte
 7504 instruct storeB(iRegIorL2I src, memory1 mem)
 7505 %{
 7506   match(Set mem (StoreB mem src));
 7507   predicate(!needs_releasing_store(n));
 7508 
 7509   ins_cost(INSN_COST);
 7510   format %{ "strb  $src, $mem\t# byte" %}
 7511 
 7512   ins_encode(aarch64_enc_strb(src, mem));
 7513 
 7514   ins_pipe(istore_reg_mem);
 7515 %}
 7516 
 7517 
 7518 instruct storeimmB0(immI0 zero, memory1 mem)
 7519 %{
 7520   match(Set mem (StoreB mem zero));
 7521   predicate(!needs_releasing_store(n));
 7522 
 7523   ins_cost(INSN_COST);
 7524   format %{ "strb rscractch2, $mem\t# byte" %}
 7525 
 7526   ins_encode(aarch64_enc_strb0(mem));
 7527 
 7528   ins_pipe(istore_mem);
 7529 %}
 7530 
 7531 // Store Char/Short
 7532 instruct storeC(iRegIorL2I src, memory2 mem)
 7533 %{
 7534   match(Set mem (StoreC mem src));
 7535   predicate(!needs_releasing_store(n));
 7536 
 7537   ins_cost(INSN_COST);
 7538   format %{ "strh  $src, $mem\t# short" %}
 7539 
 7540   ins_encode(aarch64_enc_strh(src, mem));
 7541 
 7542   ins_pipe(istore_reg_mem);
 7543 %}
 7544 
 7545 instruct storeimmC0(immI0 zero, memory2 mem)
 7546 %{
 7547   match(Set mem (StoreC mem zero));
 7548   predicate(!needs_releasing_store(n));
 7549 
 7550   ins_cost(INSN_COST);
 7551   format %{ "strh  zr, $mem\t# short" %}
 7552 
 7553   ins_encode(aarch64_enc_strh0(mem));
 7554 
 7555   ins_pipe(istore_mem);
 7556 %}
 7557 
 7558 // Store Integer
 7559 
 7560 instruct storeI(iRegIorL2I src, memory4 mem)
 7561 %{
 7562   match(Set mem(StoreI mem src));
 7563   predicate(!needs_releasing_store(n));
 7564 
 7565   ins_cost(INSN_COST);
 7566   format %{ "strw  $src, $mem\t# int" %}
 7567 
 7568   ins_encode(aarch64_enc_strw(src, mem));
 7569 
 7570   ins_pipe(istore_reg_mem);
 7571 %}
 7572 
 7573 instruct storeimmI0(immI0 zero, memory4 mem)
 7574 %{
 7575   match(Set mem(StoreI mem zero));
 7576   predicate(!needs_releasing_store(n));
 7577 
 7578   ins_cost(INSN_COST);
 7579   format %{ "strw  zr, $mem\t# int" %}
 7580 
 7581   ins_encode(aarch64_enc_strw0(mem));
 7582 
 7583   ins_pipe(istore_mem);
 7584 %}
 7585 
 7586 // Store Long (64 bit signed)
 7587 instruct storeL(iRegL src, memory8 mem)
 7588 %{
 7589   match(Set mem (StoreL mem src));
 7590   predicate(!needs_releasing_store(n));
 7591 
 7592   ins_cost(INSN_COST);
 7593   format %{ "str  $src, $mem\t# int" %}
 7594 
 7595   ins_encode(aarch64_enc_str(src, mem));
 7596 
 7597   ins_pipe(istore_reg_mem);
 7598 %}
 7599 
 7600 // Store Long (64 bit signed)
 7601 instruct storeimmL0(immL0 zero, memory8 mem)
 7602 %{
 7603   match(Set mem (StoreL mem zero));
 7604   predicate(!needs_releasing_store(n));
 7605 
 7606   ins_cost(INSN_COST);
 7607   format %{ "str  zr, $mem\t# int" %}
 7608 
 7609   ins_encode(aarch64_enc_str0(mem));
 7610 
 7611   ins_pipe(istore_mem);
 7612 %}
 7613 
 7614 // Store Pointer
 7615 instruct storeP(iRegP src, memory8 mem)
 7616 %{
 7617   match(Set mem (StoreP mem src));
 7618   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7619 
 7620   ins_cost(INSN_COST);
 7621   format %{ "str  $src, $mem\t# ptr" %}
 7622 
 7623   ins_encode(aarch64_enc_str(src, mem));
 7624 
 7625   ins_pipe(istore_reg_mem);
 7626 %}
 7627 
 7628 // Store Pointer
 7629 instruct storeimmP0(immP0 zero, memory8 mem)
 7630 %{
 7631   match(Set mem (StoreP mem zero));
 7632   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7633 
 7634   ins_cost(INSN_COST);
 7635   format %{ "str zr, $mem\t# ptr" %}
 7636 
 7637   ins_encode(aarch64_enc_str0(mem));
 7638 
 7639   ins_pipe(istore_mem);
 7640 %}
 7641 
 7642 // Store Compressed Pointer
 7643 instruct storeN(iRegN src, memory4 mem)
 7644 %{
 7645   match(Set mem (StoreN mem src));
 7646   predicate(!needs_releasing_store(n));
 7647 
 7648   ins_cost(INSN_COST);
 7649   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7650 
 7651   ins_encode(aarch64_enc_strw(src, mem));
 7652 
 7653   ins_pipe(istore_reg_mem);
 7654 %}
 7655 
 7656 instruct storeImmN0(immN0 zero, memory4 mem)
 7657 %{
 7658   match(Set mem (StoreN mem zero));
 7659   predicate(!needs_releasing_store(n));
 7660 
 7661   ins_cost(INSN_COST);
 7662   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7663 
 7664   ins_encode(aarch64_enc_strw0(mem));
 7665 
 7666   ins_pipe(istore_mem);
 7667 %}
 7668 
 7669 // Store Float
 7670 instruct storeF(vRegF src, memory4 mem)
 7671 %{
 7672   match(Set mem (StoreF mem src));
 7673   predicate(!needs_releasing_store(n));
 7674 
 7675   ins_cost(INSN_COST);
 7676   format %{ "strs  $src, $mem\t# float" %}
 7677 
 7678   ins_encode( aarch64_enc_strs(src, mem) );
 7679 
 7680   ins_pipe(pipe_class_memory);
 7681 %}
 7682 
 7683 // TODO
 7684 // implement storeImmF0 and storeFImmPacked
 7685 
 7686 // Store Double
 7687 instruct storeD(vRegD src, memory8 mem)
 7688 %{
 7689   match(Set mem (StoreD mem src));
 7690   predicate(!needs_releasing_store(n));
 7691 
 7692   ins_cost(INSN_COST);
 7693   format %{ "strd  $src, $mem\t# double" %}
 7694 
 7695   ins_encode( aarch64_enc_strd(src, mem) );
 7696 
 7697   ins_pipe(pipe_class_memory);
 7698 %}
 7699 
 7700 // Store Compressed Klass Pointer
 7701 instruct storeNKlass(iRegN src, memory4 mem)
 7702 %{
 7703   predicate(!needs_releasing_store(n));
 7704   match(Set mem (StoreNKlass mem src));
 7705 
 7706   ins_cost(INSN_COST);
 7707   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7708 
 7709   ins_encode(aarch64_enc_strw(src, mem));
 7710 
 7711   ins_pipe(istore_reg_mem);
 7712 %}
 7713 
 7714 // TODO
 7715 // implement storeImmD0 and storeDImmPacked
 7716 
 7717 // prefetch instructions
 7718 // Must be safe to execute with invalid address (cannot fault).
 7719 
 7720 instruct prefetchalloc( memory8 mem ) %{
 7721   match(PrefetchAllocation mem);
 7722 
 7723   ins_cost(INSN_COST);
 7724   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7725 
 7726   ins_encode( aarch64_enc_prefetchw(mem) );
 7727 
 7728   ins_pipe(iload_prefetch);
 7729 %}
 7730 
 7731 //  ---------------- volatile loads and stores ----------------
 7732 
 7733 // Load Byte (8 bit signed)
 7734 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7735 %{
 7736   match(Set dst (LoadB mem));
 7737 
 7738   ins_cost(VOLATILE_REF_COST);
 7739   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7740 
 7741   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7742 
 7743   ins_pipe(pipe_serial);
 7744 %}
 7745 
 7746 // Load Byte (8 bit signed) into long
 7747 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7748 %{
 7749   match(Set dst (ConvI2L (LoadB mem)));
 7750 
 7751   ins_cost(VOLATILE_REF_COST);
 7752   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7753 
 7754   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7755 
 7756   ins_pipe(pipe_serial);
 7757 %}
 7758 
 7759 // Load Byte (8 bit unsigned)
 7760 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7761 %{
 7762   match(Set dst (LoadUB mem));
 7763 
 7764   ins_cost(VOLATILE_REF_COST);
 7765   format %{ "ldarb  $dst, $mem\t# byte" %}
 7766 
 7767   ins_encode(aarch64_enc_ldarb(dst, mem));
 7768 
 7769   ins_pipe(pipe_serial);
 7770 %}
 7771 
 7772 // Load Byte (8 bit unsigned) into long
 7773 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7774 %{
 7775   match(Set dst (ConvI2L (LoadUB mem)));
 7776 
 7777   ins_cost(VOLATILE_REF_COST);
 7778   format %{ "ldarb  $dst, $mem\t# byte" %}
 7779 
 7780   ins_encode(aarch64_enc_ldarb(dst, mem));
 7781 
 7782   ins_pipe(pipe_serial);
 7783 %}
 7784 
 7785 // Load Short (16 bit signed)
 7786 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7787 %{
 7788   match(Set dst (LoadS mem));
 7789 
 7790   ins_cost(VOLATILE_REF_COST);
 7791   format %{ "ldarshw  $dst, $mem\t# short" %}
 7792 
 7793   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7794 
 7795   ins_pipe(pipe_serial);
 7796 %}
 7797 
 7798 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7799 %{
 7800   match(Set dst (LoadUS mem));
 7801 
 7802   ins_cost(VOLATILE_REF_COST);
 7803   format %{ "ldarhw  $dst, $mem\t# short" %}
 7804 
 7805   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7806 
 7807   ins_pipe(pipe_serial);
 7808 %}
 7809 
 7810 // Load Short/Char (16 bit unsigned) into long
 7811 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7812 %{
 7813   match(Set dst (ConvI2L (LoadUS mem)));
 7814 
 7815   ins_cost(VOLATILE_REF_COST);
 7816   format %{ "ldarh  $dst, $mem\t# short" %}
 7817 
 7818   ins_encode(aarch64_enc_ldarh(dst, mem));
 7819 
 7820   ins_pipe(pipe_serial);
 7821 %}
 7822 
 7823 // Load Short/Char (16 bit signed) into long
 7824 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7825 %{
 7826   match(Set dst (ConvI2L (LoadS mem)));
 7827 
 7828   ins_cost(VOLATILE_REF_COST);
 7829   format %{ "ldarh  $dst, $mem\t# short" %}
 7830 
 7831   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7832 
 7833   ins_pipe(pipe_serial);
 7834 %}
 7835 
 7836 // Load Integer (32 bit signed)
 7837 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7838 %{
 7839   match(Set dst (LoadI mem));
 7840 
 7841   ins_cost(VOLATILE_REF_COST);
 7842   format %{ "ldarw  $dst, $mem\t# int" %}
 7843 
 7844   ins_encode(aarch64_enc_ldarw(dst, mem));
 7845 
 7846   ins_pipe(pipe_serial);
 7847 %}
 7848 
 7849 // Load Integer (32 bit unsigned) into long
 7850 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7851 %{
 7852   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7853 
 7854   ins_cost(VOLATILE_REF_COST);
 7855   format %{ "ldarw  $dst, $mem\t# int" %}
 7856 
 7857   ins_encode(aarch64_enc_ldarw(dst, mem));
 7858 
 7859   ins_pipe(pipe_serial);
 7860 %}
 7861 
 7862 // Load Long (64 bit signed)
 7863 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7864 %{
 7865   match(Set dst (LoadL mem));
 7866 
 7867   ins_cost(VOLATILE_REF_COST);
 7868   format %{ "ldar  $dst, $mem\t# int" %}
 7869 
 7870   ins_encode(aarch64_enc_ldar(dst, mem));
 7871 
 7872   ins_pipe(pipe_serial);
 7873 %}
 7874 
 7875 // Load Pointer
 7876 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7877 %{
 7878   match(Set dst (LoadP mem));
 7879   predicate(n->as_Load()->barrier_data() == 0);
 7880 
 7881   ins_cost(VOLATILE_REF_COST);
 7882   format %{ "ldar  $dst, $mem\t# ptr" %}
 7883 
 7884   ins_encode(aarch64_enc_ldar(dst, mem));
 7885 
 7886   ins_pipe(pipe_serial);
 7887 %}
 7888 
 7889 // Load Compressed Pointer
 7890 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7891 %{
 7892   match(Set dst (LoadN mem));
 7893 
 7894   ins_cost(VOLATILE_REF_COST);
 7895   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7896 
 7897   ins_encode(aarch64_enc_ldarw(dst, mem));
 7898 
 7899   ins_pipe(pipe_serial);
 7900 %}
 7901 
 7902 // Load Float
 7903 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7904 %{
 7905   match(Set dst (LoadF mem));
 7906 
 7907   ins_cost(VOLATILE_REF_COST);
 7908   format %{ "ldars  $dst, $mem\t# float" %}
 7909 
 7910   ins_encode( aarch64_enc_fldars(dst, mem) );
 7911 
 7912   ins_pipe(pipe_serial);
 7913 %}
 7914 
 7915 // Load Double
 7916 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7917 %{
 7918   match(Set dst (LoadD mem));
 7919 
 7920   ins_cost(VOLATILE_REF_COST);
 7921   format %{ "ldard  $dst, $mem\t# double" %}
 7922 
 7923   ins_encode( aarch64_enc_fldard(dst, mem) );
 7924 
 7925   ins_pipe(pipe_serial);
 7926 %}
 7927 
 7928 // Store Byte
 7929 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7930 %{
 7931   match(Set mem (StoreB mem src));
 7932 
 7933   ins_cost(VOLATILE_REF_COST);
 7934   format %{ "stlrb  $src, $mem\t# byte" %}
 7935 
 7936   ins_encode(aarch64_enc_stlrb(src, mem));
 7937 
 7938   ins_pipe(pipe_class_memory);
 7939 %}
 7940 
 7941 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7942 %{
 7943   match(Set mem (StoreB mem zero));
 7944 
 7945   ins_cost(VOLATILE_REF_COST);
 7946   format %{ "stlrb  zr, $mem\t# byte" %}
 7947 
 7948   ins_encode(aarch64_enc_stlrb0(mem));
 7949 
 7950   ins_pipe(pipe_class_memory);
 7951 %}
 7952 
 7953 // Store Char/Short
 7954 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7955 %{
 7956   match(Set mem (StoreC mem src));
 7957 
 7958   ins_cost(VOLATILE_REF_COST);
 7959   format %{ "stlrh  $src, $mem\t# short" %}
 7960 
 7961   ins_encode(aarch64_enc_stlrh(src, mem));
 7962 
 7963   ins_pipe(pipe_class_memory);
 7964 %}
 7965 
 7966 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7967 %{
 7968   match(Set mem (StoreC mem zero));
 7969 
 7970   ins_cost(VOLATILE_REF_COST);
 7971   format %{ "stlrh  zr, $mem\t# short" %}
 7972 
 7973   ins_encode(aarch64_enc_stlrh0(mem));
 7974 
 7975   ins_pipe(pipe_class_memory);
 7976 %}
 7977 
 7978 // Store Integer
 7979 
 7980 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7981 %{
 7982   match(Set mem(StoreI mem src));
 7983 
 7984   ins_cost(VOLATILE_REF_COST);
 7985   format %{ "stlrw  $src, $mem\t# int" %}
 7986 
 7987   ins_encode(aarch64_enc_stlrw(src, mem));
 7988 
 7989   ins_pipe(pipe_class_memory);
 7990 %}
 7991 
 7992 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7993 %{
 7994   match(Set mem(StoreI mem zero));
 7995 
 7996   ins_cost(VOLATILE_REF_COST);
 7997   format %{ "stlrw  zr, $mem\t# int" %}
 7998 
 7999   ins_encode(aarch64_enc_stlrw0(mem));
 8000 
 8001   ins_pipe(pipe_class_memory);
 8002 %}
 8003 
 8004 // Store Long (64 bit signed)
 8005 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 8006 %{
 8007   match(Set mem (StoreL mem src));
 8008 
 8009   ins_cost(VOLATILE_REF_COST);
 8010   format %{ "stlr  $src, $mem\t# int" %}
 8011 
 8012   ins_encode(aarch64_enc_stlr(src, mem));
 8013 
 8014   ins_pipe(pipe_class_memory);
 8015 %}
 8016 
 8017 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8018 %{
 8019   match(Set mem (StoreL mem zero));
 8020 
 8021   ins_cost(VOLATILE_REF_COST);
 8022   format %{ "stlr  zr, $mem\t# int" %}
 8023 
 8024   ins_encode(aarch64_enc_stlr0(mem));
 8025 
 8026   ins_pipe(pipe_class_memory);
 8027 %}
 8028 
 8029 // Store Pointer
 8030 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8031 %{
 8032   match(Set mem (StoreP mem src));
 8033   predicate(n->as_Store()->barrier_data() == 0);
 8034 
 8035   ins_cost(VOLATILE_REF_COST);
 8036   format %{ "stlr  $src, $mem\t# ptr" %}
 8037 
 8038   ins_encode(aarch64_enc_stlr(src, mem));
 8039 
 8040   ins_pipe(pipe_class_memory);
 8041 %}
 8042 
 8043 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8044 %{
 8045   match(Set mem (StoreP mem zero));
 8046   predicate(n->as_Store()->barrier_data() == 0);
 8047 
 8048   ins_cost(VOLATILE_REF_COST);
 8049   format %{ "stlr  zr, $mem\t# ptr" %}
 8050 
 8051   ins_encode(aarch64_enc_stlr0(mem));
 8052 
 8053   ins_pipe(pipe_class_memory);
 8054 %}
 8055 
 8056 // Store Compressed Pointer
 8057 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8058 %{
 8059   match(Set mem (StoreN mem src));
 8060 
 8061   ins_cost(VOLATILE_REF_COST);
 8062   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8063 
 8064   ins_encode(aarch64_enc_stlrw(src, mem));
 8065 
 8066   ins_pipe(pipe_class_memory);
 8067 %}
 8068 
 8069 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8070 %{
 8071   match(Set mem (StoreN mem zero));
 8072 
 8073   ins_cost(VOLATILE_REF_COST);
 8074   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8075 
 8076   ins_encode(aarch64_enc_stlrw0(mem));
 8077 
 8078   ins_pipe(pipe_class_memory);
 8079 %}
 8080 
 8081 // Store Float
 8082 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8083 %{
 8084   match(Set mem (StoreF mem src));
 8085 
 8086   ins_cost(VOLATILE_REF_COST);
 8087   format %{ "stlrs  $src, $mem\t# float" %}
 8088 
 8089   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8090 
 8091   ins_pipe(pipe_class_memory);
 8092 %}
 8093 
 8094 // TODO
 8095 // implement storeImmF0 and storeFImmPacked
 8096 
 8097 // Store Double
 8098 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8099 %{
 8100   match(Set mem (StoreD mem src));
 8101 
 8102   ins_cost(VOLATILE_REF_COST);
 8103   format %{ "stlrd  $src, $mem\t# double" %}
 8104 
 8105   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8106 
 8107   ins_pipe(pipe_class_memory);
 8108 %}
 8109 
 8110 //  ---------------- end of volatile loads and stores ----------------
 8111 
 8112 instruct cacheWB(indirect addr)
 8113 %{
 8114   predicate(VM_Version::supports_data_cache_line_flush());
 8115   match(CacheWB addr);
 8116 
 8117   ins_cost(100);
 8118   format %{"cache wb $addr" %}
 8119   ins_encode %{
 8120     assert($addr->index_position() < 0, "should be");
 8121     assert($addr$$disp == 0, "should be");
 8122     __ cache_wb(Address($addr$$base$$Register, 0));
 8123   %}
 8124   ins_pipe(pipe_slow); // XXX
 8125 %}
 8126 
 8127 instruct cacheWBPreSync()
 8128 %{
 8129   predicate(VM_Version::supports_data_cache_line_flush());
 8130   match(CacheWBPreSync);
 8131 
 8132   ins_cost(100);
 8133   format %{"cache wb presync" %}
 8134   ins_encode %{
 8135     __ cache_wbsync(true);
 8136   %}
 8137   ins_pipe(pipe_slow); // XXX
 8138 %}
 8139 
 8140 instruct cacheWBPostSync()
 8141 %{
 8142   predicate(VM_Version::supports_data_cache_line_flush());
 8143   match(CacheWBPostSync);
 8144 
 8145   ins_cost(100);
 8146   format %{"cache wb postsync" %}
 8147   ins_encode %{
 8148     __ cache_wbsync(false);
 8149   %}
 8150   ins_pipe(pipe_slow); // XXX
 8151 %}
 8152 
 8153 // ============================================================================
 8154 // BSWAP Instructions
 8155 
 8156 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8157   match(Set dst (ReverseBytesI src));
 8158 
 8159   ins_cost(INSN_COST);
 8160   format %{ "revw  $dst, $src" %}
 8161 
 8162   ins_encode %{
 8163     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8164   %}
 8165 
 8166   ins_pipe(ialu_reg);
 8167 %}
 8168 
 8169 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8170   match(Set dst (ReverseBytesL src));
 8171 
 8172   ins_cost(INSN_COST);
 8173   format %{ "rev  $dst, $src" %}
 8174 
 8175   ins_encode %{
 8176     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8177   %}
 8178 
 8179   ins_pipe(ialu_reg);
 8180 %}
 8181 
 8182 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8183   match(Set dst (ReverseBytesUS src));
 8184 
 8185   ins_cost(INSN_COST);
 8186   format %{ "rev16w  $dst, $src" %}
 8187 
 8188   ins_encode %{
 8189     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8190   %}
 8191 
 8192   ins_pipe(ialu_reg);
 8193 %}
 8194 
 8195 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8196   match(Set dst (ReverseBytesS src));
 8197 
 8198   ins_cost(INSN_COST);
 8199   format %{ "rev16w  $dst, $src\n\t"
 8200             "sbfmw $dst, $dst, #0, #15" %}
 8201 
 8202   ins_encode %{
 8203     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8204     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8205   %}
 8206 
 8207   ins_pipe(ialu_reg);
 8208 %}
 8209 
 8210 // ============================================================================
 8211 // Zero Count Instructions
 8212 
 8213 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8214   match(Set dst (CountLeadingZerosI src));
 8215 
 8216   ins_cost(INSN_COST);
 8217   format %{ "clzw  $dst, $src" %}
 8218   ins_encode %{
 8219     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8220   %}
 8221 
 8222   ins_pipe(ialu_reg);
 8223 %}
 8224 
 8225 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8226   match(Set dst (CountLeadingZerosL src));
 8227 
 8228   ins_cost(INSN_COST);
 8229   format %{ "clz   $dst, $src" %}
 8230   ins_encode %{
 8231     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8232   %}
 8233 
 8234   ins_pipe(ialu_reg);
 8235 %}
 8236 
 8237 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8238   match(Set dst (CountTrailingZerosI src));
 8239 
 8240   ins_cost(INSN_COST * 2);
 8241   format %{ "rbitw  $dst, $src\n\t"
 8242             "clzw   $dst, $dst" %}
 8243   ins_encode %{
 8244     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8245     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8246   %}
 8247 
 8248   ins_pipe(ialu_reg);
 8249 %}
 8250 
 8251 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8252   match(Set dst (CountTrailingZerosL src));
 8253 
 8254   ins_cost(INSN_COST * 2);
 8255   format %{ "rbit   $dst, $src\n\t"
 8256             "clz    $dst, $dst" %}
 8257   ins_encode %{
 8258     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8259     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8260   %}
 8261 
 8262   ins_pipe(ialu_reg);
 8263 %}
 8264 
 8265 //---------- Population Count Instructions -------------------------------------
 8266 //
 8267 
 8268 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8269   match(Set dst (PopCountI src));
 8270   effect(TEMP tmp);
 8271   ins_cost(INSN_COST * 13);
 8272 
 8273   format %{ "movw   $src, $src\n\t"
 8274             "mov    $tmp, $src\t# vector (1D)\n\t"
 8275             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8276             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8277             "mov    $dst, $tmp\t# vector (1D)" %}
 8278   ins_encode %{
 8279     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8280     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8281     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8282     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8283     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8284   %}
 8285 
 8286   ins_pipe(pipe_class_default);
 8287 %}
 8288 
 8289 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8290   match(Set dst (PopCountI (LoadI mem)));
 8291   effect(TEMP tmp);
 8292   ins_cost(INSN_COST * 13);
 8293 
 8294   format %{ "ldrs   $tmp, $mem\n\t"
 8295             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8296             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8297             "mov    $dst, $tmp\t# vector (1D)" %}
 8298   ins_encode %{
 8299     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8300     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8301               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8302     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8303     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8304     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8305   %}
 8306 
 8307   ins_pipe(pipe_class_default);
 8308 %}
 8309 
 8310 // Note: Long.bitCount(long) returns an int.
 8311 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8312   match(Set dst (PopCountL src));
 8313   effect(TEMP tmp);
 8314   ins_cost(INSN_COST * 13);
 8315 
 8316   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8317             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8318             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8319             "mov    $dst, $tmp\t# vector (1D)" %}
 8320   ins_encode %{
 8321     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8322     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8323     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8324     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8325   %}
 8326 
 8327   ins_pipe(pipe_class_default);
 8328 %}
 8329 
 8330 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8331   match(Set dst (PopCountL (LoadL mem)));
 8332   effect(TEMP tmp);
 8333   ins_cost(INSN_COST * 13);
 8334 
 8335   format %{ "ldrd   $tmp, $mem\n\t"
 8336             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8337             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8338             "mov    $dst, $tmp\t# vector (1D)" %}
 8339   ins_encode %{
 8340     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8341     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8342               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8343     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8344     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8345     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8346   %}
 8347 
 8348   ins_pipe(pipe_class_default);
 8349 %}
 8350 
 8351 // ============================================================================
 8352 // MemBar Instruction
 8353 
 8354 instruct load_fence() %{
 8355   match(LoadFence);
 8356   ins_cost(VOLATILE_REF_COST);
 8357 
 8358   format %{ "load_fence" %}
 8359 
 8360   ins_encode %{
 8361     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8362   %}
 8363   ins_pipe(pipe_serial);
 8364 %}
 8365 
 8366 instruct unnecessary_membar_acquire() %{
 8367   predicate(unnecessary_acquire(n));
 8368   match(MemBarAcquire);
 8369   ins_cost(0);
 8370 
 8371   format %{ "membar_acquire (elided)" %}
 8372 
 8373   ins_encode %{
 8374     __ block_comment("membar_acquire (elided)");
 8375   %}
 8376 
 8377   ins_pipe(pipe_class_empty);
 8378 %}
 8379 
 8380 instruct membar_acquire() %{
 8381   match(MemBarAcquire);
 8382   ins_cost(VOLATILE_REF_COST);
 8383 
 8384   format %{ "membar_acquire\n\t"
 8385             "dmb ish" %}
 8386 
 8387   ins_encode %{
 8388     __ block_comment("membar_acquire");
 8389     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8390   %}
 8391 
 8392   ins_pipe(pipe_serial);
 8393 %}
 8394 
 8395 
 8396 instruct membar_acquire_lock() %{
 8397   match(MemBarAcquireLock);
 8398   ins_cost(VOLATILE_REF_COST);
 8399 
 8400   format %{ "membar_acquire_lock (elided)" %}
 8401 
 8402   ins_encode %{
 8403     __ block_comment("membar_acquire_lock (elided)");
 8404   %}
 8405 
 8406   ins_pipe(pipe_serial);
 8407 %}
 8408 
 8409 instruct store_fence() %{
 8410   match(StoreFence);
 8411   ins_cost(VOLATILE_REF_COST);
 8412 
 8413   format %{ "store_fence" %}
 8414 
 8415   ins_encode %{
 8416     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8417   %}
 8418   ins_pipe(pipe_serial);
 8419 %}
 8420 
 8421 instruct unnecessary_membar_release() %{
 8422   predicate(unnecessary_release(n));
 8423   match(MemBarRelease);
 8424   ins_cost(0);
 8425 
 8426   format %{ "membar_release (elided)" %}
 8427 
 8428   ins_encode %{
 8429     __ block_comment("membar_release (elided)");
 8430   %}
 8431   ins_pipe(pipe_serial);
 8432 %}
 8433 
 8434 instruct membar_release() %{
 8435   match(MemBarRelease);
 8436   ins_cost(VOLATILE_REF_COST);
 8437 
 8438   format %{ "membar_release\n\t"
 8439             "dmb ish" %}
 8440 
 8441   ins_encode %{
 8442     __ block_comment("membar_release");
 8443     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8444   %}
 8445   ins_pipe(pipe_serial);
 8446 %}
 8447 
 8448 instruct membar_storestore() %{
 8449   match(MemBarStoreStore);
 8450   match(StoreStoreFence);
 8451   ins_cost(VOLATILE_REF_COST);
 8452 
 8453   format %{ "MEMBAR-store-store" %}
 8454 
 8455   ins_encode %{
 8456     __ membar(Assembler::StoreStore);
 8457   %}
 8458   ins_pipe(pipe_serial);
 8459 %}
 8460 
 8461 instruct membar_release_lock() %{
 8462   match(MemBarReleaseLock);
 8463   ins_cost(VOLATILE_REF_COST);
 8464 
 8465   format %{ "membar_release_lock (elided)" %}
 8466 
 8467   ins_encode %{
 8468     __ block_comment("membar_release_lock (elided)");
 8469   %}
 8470 
 8471   ins_pipe(pipe_serial);
 8472 %}
 8473 
 8474 instruct unnecessary_membar_volatile() %{
 8475   predicate(unnecessary_volatile(n));
 8476   match(MemBarVolatile);
 8477   ins_cost(0);
 8478 
 8479   format %{ "membar_volatile (elided)" %}
 8480 
 8481   ins_encode %{
 8482     __ block_comment("membar_volatile (elided)");
 8483   %}
 8484 
 8485   ins_pipe(pipe_serial);
 8486 %}
 8487 
 8488 instruct membar_volatile() %{
 8489   match(MemBarVolatile);
 8490   ins_cost(VOLATILE_REF_COST*100);
 8491 
 8492   format %{ "membar_volatile\n\t"
 8493              "dmb ish"%}
 8494 
 8495   ins_encode %{
 8496     __ block_comment("membar_volatile");
 8497     __ membar(Assembler::StoreLoad);
 8498   %}
 8499 
 8500   ins_pipe(pipe_serial);
 8501 %}
 8502 
 8503 // ============================================================================
 8504 // Cast/Convert Instructions
 8505 
 8506 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8507   match(Set dst (CastX2P src));
 8508 
 8509   ins_cost(INSN_COST);
 8510   format %{ "mov $dst, $src\t# long -> ptr" %}
 8511 
 8512   ins_encode %{
 8513     if ($dst$$reg != $src$$reg) {
 8514       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8515     }
 8516   %}
 8517 
 8518   ins_pipe(ialu_reg);
 8519 %}
 8520 
 8521 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8522   match(Set dst (CastP2X src));
 8523 
 8524   ins_cost(INSN_COST);
 8525   format %{ "mov $dst, $src\t# ptr -> long" %}
 8526 
 8527   ins_encode %{
 8528     if ($dst$$reg != $src$$reg) {
 8529       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8530     }
 8531   %}
 8532 
 8533   ins_pipe(ialu_reg);
 8534 %}
 8535 
 8536 // Convert oop into int for vectors alignment masking
 8537 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8538   match(Set dst (ConvL2I (CastP2X src)));
 8539 
 8540   ins_cost(INSN_COST);
 8541   format %{ "movw $dst, $src\t# ptr -> int" %}
 8542   ins_encode %{
 8543     __ movw($dst$$Register, $src$$Register);
 8544   %}
 8545 
 8546   ins_pipe(ialu_reg);
 8547 %}
 8548 
 8549 // Convert compressed oop into int for vectors alignment masking
 8550 // in case of 32bit oops (heap < 4Gb).
 8551 instruct convN2I(iRegINoSp dst, iRegN src)
 8552 %{
 8553   predicate(CompressedOops::shift() == 0);
 8554   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8555 
 8556   ins_cost(INSN_COST);
 8557   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8558   ins_encode %{
 8559     __ movw($dst$$Register, $src$$Register);
 8560   %}
 8561 
 8562   ins_pipe(ialu_reg);
 8563 %}
 8564 
 8565 
 8566 // Convert oop pointer into compressed form
 8567 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8568   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8569   match(Set dst (EncodeP src));
 8570   effect(KILL cr);
 8571   ins_cost(INSN_COST * 3);
 8572   format %{ "encode_heap_oop $dst, $src" %}
 8573   ins_encode %{
 8574     Register s = $src$$Register;
 8575     Register d = $dst$$Register;
 8576     __ encode_heap_oop(d, s);
 8577   %}
 8578   ins_pipe(ialu_reg);
 8579 %}
 8580 
 8581 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8582   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8583   match(Set dst (EncodeP src));
 8584   ins_cost(INSN_COST * 3);
 8585   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8586   ins_encode %{
 8587     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8588   %}
 8589   ins_pipe(ialu_reg);
 8590 %}
 8591 
 8592 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8593   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8594             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8595   match(Set dst (DecodeN src));
 8596   ins_cost(INSN_COST * 3);
 8597   format %{ "decode_heap_oop $dst, $src" %}
 8598   ins_encode %{
 8599     Register s = $src$$Register;
 8600     Register d = $dst$$Register;
 8601     __ decode_heap_oop(d, s);
 8602   %}
 8603   ins_pipe(ialu_reg);
 8604 %}
 8605 
 8606 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8607   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8608             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8609   match(Set dst (DecodeN src));
 8610   ins_cost(INSN_COST * 3);
 8611   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8612   ins_encode %{
 8613     Register s = $src$$Register;
 8614     Register d = $dst$$Register;
 8615     __ decode_heap_oop_not_null(d, s);
 8616   %}
 8617   ins_pipe(ialu_reg);
 8618 %}
 8619 
 8620 // n.b. AArch64 implementations of encode_klass_not_null and
 8621 // decode_klass_not_null do not modify the flags register so, unlike
 8622 // Intel, we don't kill CR as a side effect here
 8623 
 8624 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8625   match(Set dst (EncodePKlass src));
 8626 
 8627   ins_cost(INSN_COST * 3);
 8628   format %{ "encode_klass_not_null $dst,$src" %}
 8629 
 8630   ins_encode %{
 8631     Register src_reg = as_Register($src$$reg);
 8632     Register dst_reg = as_Register($dst$$reg);
 8633     __ encode_klass_not_null(dst_reg, src_reg);
 8634   %}
 8635 
 8636    ins_pipe(ialu_reg);
 8637 %}
 8638 
 8639 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8640   match(Set dst (DecodeNKlass src));
 8641 
 8642   ins_cost(INSN_COST * 3);
 8643   format %{ "decode_klass_not_null $dst,$src" %}
 8644 
 8645   ins_encode %{
 8646     Register src_reg = as_Register($src$$reg);
 8647     Register dst_reg = as_Register($dst$$reg);
 8648     if (dst_reg != src_reg) {
 8649       __ decode_klass_not_null(dst_reg, src_reg);
 8650     } else {
 8651       __ decode_klass_not_null(dst_reg);
 8652     }
 8653   %}
 8654 
 8655    ins_pipe(ialu_reg);
 8656 %}
 8657 
 8658 instruct checkCastPP(iRegPNoSp dst)
 8659 %{
 8660   match(Set dst (CheckCastPP dst));
 8661 
 8662   size(0);
 8663   format %{ "# checkcastPP of $dst" %}
 8664   ins_encode(/* empty encoding */);
 8665   ins_pipe(pipe_class_empty);
 8666 %}
 8667 
 8668 instruct castPP(iRegPNoSp dst)
 8669 %{
 8670   match(Set dst (CastPP dst));
 8671 
 8672   size(0);
 8673   format %{ "# castPP of $dst" %}
 8674   ins_encode(/* empty encoding */);
 8675   ins_pipe(pipe_class_empty);
 8676 %}
 8677 
 8678 instruct castII(iRegI dst)
 8679 %{
 8680   match(Set dst (CastII dst));
 8681 
 8682   size(0);
 8683   format %{ "# castII of $dst" %}
 8684   ins_encode(/* empty encoding */);
 8685   ins_cost(0);
 8686   ins_pipe(pipe_class_empty);
 8687 %}
 8688 
 8689 instruct castLL(iRegL dst)
 8690 %{
 8691   match(Set dst (CastLL dst));
 8692 
 8693   size(0);
 8694   format %{ "# castLL of $dst" %}
 8695   ins_encode(/* empty encoding */);
 8696   ins_cost(0);
 8697   ins_pipe(pipe_class_empty);
 8698 %}
 8699 
 8700 instruct castFF(vRegF dst)
 8701 %{
 8702   match(Set dst (CastFF dst));
 8703 
 8704   size(0);
 8705   format %{ "# castFF of $dst" %}
 8706   ins_encode(/* empty encoding */);
 8707   ins_cost(0);
 8708   ins_pipe(pipe_class_empty);
 8709 %}
 8710 
 8711 instruct castDD(vRegD dst)
 8712 %{
 8713   match(Set dst (CastDD dst));
 8714 
 8715   size(0);
 8716   format %{ "# castDD of $dst" %}
 8717   ins_encode(/* empty encoding */);
 8718   ins_cost(0);
 8719   ins_pipe(pipe_class_empty);
 8720 %}
 8721 
 8722 instruct castVV(vReg dst)
 8723 %{
 8724   match(Set dst (CastVV dst));
 8725 
 8726   size(0);
 8727   format %{ "# castVV of $dst" %}
 8728   ins_encode(/* empty encoding */);
 8729   ins_cost(0);
 8730   ins_pipe(pipe_class_empty);
 8731 %}
 8732 
 8733 instruct castVVMask(pRegGov dst)
 8734 %{
 8735   match(Set dst (CastVV dst));
 8736 
 8737   size(0);
 8738   format %{ "# castVV of $dst" %}
 8739   ins_encode(/* empty encoding */);
 8740   ins_cost(0);
 8741   ins_pipe(pipe_class_empty);
 8742 %}
 8743 
 8744 // ============================================================================
 8745 // Atomic operation instructions
 8746 //
 8747 
 8748 // standard CompareAndSwapX when we are using barriers
 8749 // these have higher priority than the rules selected by a predicate
 8750 
 8751 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8752 // can't match them
 8753 
 8754 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8755 
 8756   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8757   ins_cost(2 * VOLATILE_REF_COST);
 8758 
 8759   effect(KILL cr);
 8760 
 8761   format %{
 8762     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8763     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8764   %}
 8765 
 8766   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8767             aarch64_enc_cset_eq(res));
 8768 
 8769   ins_pipe(pipe_slow);
 8770 %}
 8771 
 8772 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8773 
 8774   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8775   ins_cost(2 * VOLATILE_REF_COST);
 8776 
 8777   effect(KILL cr);
 8778 
 8779   format %{
 8780     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8781     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8782   %}
 8783 
 8784   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8785             aarch64_enc_cset_eq(res));
 8786 
 8787   ins_pipe(pipe_slow);
 8788 %}
 8789 
 8790 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8791 
 8792   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8793   ins_cost(2 * VOLATILE_REF_COST);
 8794 
 8795   effect(KILL cr);
 8796 
 8797  format %{
 8798     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8799     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8800  %}
 8801 
 8802  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8803             aarch64_enc_cset_eq(res));
 8804 
 8805   ins_pipe(pipe_slow);
 8806 %}
 8807 
 8808 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8809 
 8810   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8811   ins_cost(2 * VOLATILE_REF_COST);
 8812 
 8813   effect(KILL cr);
 8814 
 8815  format %{
 8816     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8817     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8818  %}
 8819 
 8820  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8821             aarch64_enc_cset_eq(res));
 8822 
 8823   ins_pipe(pipe_slow);
 8824 %}
 8825 
 8826 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8827 
 8828   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8829   predicate(n->as_LoadStore()->barrier_data() == 0);
 8830   ins_cost(2 * VOLATILE_REF_COST);
 8831 
 8832   effect(KILL cr);
 8833 
 8834  format %{
 8835     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8836     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8837  %}
 8838 
 8839  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8840             aarch64_enc_cset_eq(res));
 8841 
 8842   ins_pipe(pipe_slow);
 8843 %}
 8844 
 8845 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8846 
 8847   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8848   ins_cost(2 * VOLATILE_REF_COST);
 8849 
 8850   effect(KILL cr);
 8851 
 8852  format %{
 8853     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8854     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8855  %}
 8856 
 8857  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8858             aarch64_enc_cset_eq(res));
 8859 
 8860   ins_pipe(pipe_slow);
 8861 %}
 8862 
 8863 // alternative CompareAndSwapX when we are eliding barriers
 8864 
 8865 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8866 
 8867   predicate(needs_acquiring_load_exclusive(n));
 8868   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8869   ins_cost(VOLATILE_REF_COST);
 8870 
 8871   effect(KILL cr);
 8872 
 8873   format %{
 8874     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8875     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8876   %}
 8877 
 8878   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8879             aarch64_enc_cset_eq(res));
 8880 
 8881   ins_pipe(pipe_slow);
 8882 %}
 8883 
 8884 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8885 
 8886   predicate(needs_acquiring_load_exclusive(n));
 8887   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8888   ins_cost(VOLATILE_REF_COST);
 8889 
 8890   effect(KILL cr);
 8891 
 8892   format %{
 8893     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8894     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8895   %}
 8896 
 8897   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8898             aarch64_enc_cset_eq(res));
 8899 
 8900   ins_pipe(pipe_slow);
 8901 %}
 8902 
 8903 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8904 
 8905   predicate(needs_acquiring_load_exclusive(n));
 8906   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8907   ins_cost(VOLATILE_REF_COST);
 8908 
 8909   effect(KILL cr);
 8910 
 8911  format %{
 8912     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8913     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8914  %}
 8915 
 8916  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8917             aarch64_enc_cset_eq(res));
 8918 
 8919   ins_pipe(pipe_slow);
 8920 %}
 8921 
 8922 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8923 
 8924   predicate(needs_acquiring_load_exclusive(n));
 8925   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8926   ins_cost(VOLATILE_REF_COST);
 8927 
 8928   effect(KILL cr);
 8929 
 8930  format %{
 8931     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8932     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8933  %}
 8934 
 8935  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8936             aarch64_enc_cset_eq(res));
 8937 
 8938   ins_pipe(pipe_slow);
 8939 %}
 8940 
 8941 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8942 
 8943   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8944   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8945   ins_cost(VOLATILE_REF_COST);
 8946 
 8947   effect(KILL cr);
 8948 
 8949  format %{
 8950     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8951     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8952  %}
 8953 
 8954  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8955             aarch64_enc_cset_eq(res));
 8956 
 8957   ins_pipe(pipe_slow);
 8958 %}
 8959 
 8960 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8961 
 8962   predicate(needs_acquiring_load_exclusive(n));
 8963   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8964   ins_cost(VOLATILE_REF_COST);
 8965 
 8966   effect(KILL cr);
 8967 
 8968  format %{
 8969     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8970     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8971  %}
 8972 
 8973  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8974             aarch64_enc_cset_eq(res));
 8975 
 8976   ins_pipe(pipe_slow);
 8977 %}
 8978 
 8979 
 8980 // ---------------------------------------------------------------------
 8981 
 8982 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8983 
 8984 // Sundry CAS operations.  Note that release is always true,
 8985 // regardless of the memory ordering of the CAS.  This is because we
 8986 // need the volatile case to be sequentially consistent but there is
 8987 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8988 // can't check the type of memory ordering here, so we always emit a
 8989 // STLXR.
 8990 
 8991 // This section is generated from aarch64_ad_cas.m4
 8992 
 8993 
 8994 
 8995 // This pattern is generated automatically from cas.m4.
 8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8997 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8998 
 8999   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9000   ins_cost(2 * VOLATILE_REF_COST);
 9001   effect(TEMP_DEF res, KILL cr);
 9002   format %{
 9003     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9004   %}
 9005   ins_encode %{
 9006     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9007                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9008                /*weak*/ false, $res$$Register);
 9009     __ sxtbw($res$$Register, $res$$Register);
 9010   %}
 9011   ins_pipe(pipe_slow);
 9012 %}
 9013 
 9014 // This pattern is generated automatically from cas.m4.
 9015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9016 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9017 
 9018   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9019   ins_cost(2 * VOLATILE_REF_COST);
 9020   effect(TEMP_DEF res, KILL cr);
 9021   format %{
 9022     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9023   %}
 9024   ins_encode %{
 9025     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9026                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9027                /*weak*/ false, $res$$Register);
 9028     __ sxthw($res$$Register, $res$$Register);
 9029   %}
 9030   ins_pipe(pipe_slow);
 9031 %}
 9032 
 9033 // This pattern is generated automatically from cas.m4.
 9034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9035 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9036 
 9037   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9038   ins_cost(2 * VOLATILE_REF_COST);
 9039   effect(TEMP_DEF res, KILL cr);
 9040   format %{
 9041     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9042   %}
 9043   ins_encode %{
 9044     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9045                Assembler::word, /*acquire*/ false, /*release*/ true,
 9046                /*weak*/ false, $res$$Register);
 9047   %}
 9048   ins_pipe(pipe_slow);
 9049 %}
 9050 
 9051 // This pattern is generated automatically from cas.m4.
 9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9053 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9054 
 9055   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9056   ins_cost(2 * VOLATILE_REF_COST);
 9057   effect(TEMP_DEF res, KILL cr);
 9058   format %{
 9059     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9060   %}
 9061   ins_encode %{
 9062     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9063                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9064                /*weak*/ false, $res$$Register);
 9065   %}
 9066   ins_pipe(pipe_slow);
 9067 %}
 9068 
 9069 // This pattern is generated automatically from cas.m4.
 9070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9071 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9072 
 9073   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9074   ins_cost(2 * VOLATILE_REF_COST);
 9075   effect(TEMP_DEF res, KILL cr);
 9076   format %{
 9077     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9078   %}
 9079   ins_encode %{
 9080     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9081                Assembler::word, /*acquire*/ false, /*release*/ true,
 9082                /*weak*/ false, $res$$Register);
 9083   %}
 9084   ins_pipe(pipe_slow);
 9085 %}
 9086 
 9087 // This pattern is generated automatically from cas.m4.
 9088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9089 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9090   predicate(n->as_LoadStore()->barrier_data() == 0);
 9091   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9092   ins_cost(2 * VOLATILE_REF_COST);
 9093   effect(TEMP_DEF res, KILL cr);
 9094   format %{
 9095     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9096   %}
 9097   ins_encode %{
 9098     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9099                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9100                /*weak*/ false, $res$$Register);
 9101   %}
 9102   ins_pipe(pipe_slow);
 9103 %}
 9104 
 9105 // This pattern is generated automatically from cas.m4.
 9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9107 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9108   predicate(needs_acquiring_load_exclusive(n));
 9109   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9110   ins_cost(VOLATILE_REF_COST);
 9111   effect(TEMP_DEF res, KILL cr);
 9112   format %{
 9113     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9114   %}
 9115   ins_encode %{
 9116     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9117                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9118                /*weak*/ false, $res$$Register);
 9119     __ sxtbw($res$$Register, $res$$Register);
 9120   %}
 9121   ins_pipe(pipe_slow);
 9122 %}
 9123 
 9124 // This pattern is generated automatically from cas.m4.
 9125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9126 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9127   predicate(needs_acquiring_load_exclusive(n));
 9128   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9129   ins_cost(VOLATILE_REF_COST);
 9130   effect(TEMP_DEF res, KILL cr);
 9131   format %{
 9132     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9133   %}
 9134   ins_encode %{
 9135     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9136                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9137                /*weak*/ false, $res$$Register);
 9138     __ sxthw($res$$Register, $res$$Register);
 9139   %}
 9140   ins_pipe(pipe_slow);
 9141 %}
 9142 
 9143 // This pattern is generated automatically from cas.m4.
 9144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9145 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9146   predicate(needs_acquiring_load_exclusive(n));
 9147   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9148   ins_cost(VOLATILE_REF_COST);
 9149   effect(TEMP_DEF res, KILL cr);
 9150   format %{
 9151     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9152   %}
 9153   ins_encode %{
 9154     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9155                Assembler::word, /*acquire*/ true, /*release*/ true,
 9156                /*weak*/ false, $res$$Register);
 9157   %}
 9158   ins_pipe(pipe_slow);
 9159 %}
 9160 
 9161 // This pattern is generated automatically from cas.m4.
 9162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9163 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9164   predicate(needs_acquiring_load_exclusive(n));
 9165   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9166   ins_cost(VOLATILE_REF_COST);
 9167   effect(TEMP_DEF res, KILL cr);
 9168   format %{
 9169     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9170   %}
 9171   ins_encode %{
 9172     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9173                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9174                /*weak*/ false, $res$$Register);
 9175   %}
 9176   ins_pipe(pipe_slow);
 9177 %}
 9178 
 9179 // This pattern is generated automatically from cas.m4.
 9180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9181 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9182   predicate(needs_acquiring_load_exclusive(n));
 9183   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9184   ins_cost(VOLATILE_REF_COST);
 9185   effect(TEMP_DEF res, KILL cr);
 9186   format %{
 9187     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9188   %}
 9189   ins_encode %{
 9190     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9191                Assembler::word, /*acquire*/ true, /*release*/ true,
 9192                /*weak*/ false, $res$$Register);
 9193   %}
 9194   ins_pipe(pipe_slow);
 9195 %}
 9196 
 9197 // This pattern is generated automatically from cas.m4.
 9198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9199 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9200   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9201   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9202   ins_cost(VOLATILE_REF_COST);
 9203   effect(TEMP_DEF res, KILL cr);
 9204   format %{
 9205     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9206   %}
 9207   ins_encode %{
 9208     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9209                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9210                /*weak*/ false, $res$$Register);
 9211   %}
 9212   ins_pipe(pipe_slow);
 9213 %}
 9214 
 9215 // This pattern is generated automatically from cas.m4.
 9216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9217 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9218 
 9219   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9220   ins_cost(2 * VOLATILE_REF_COST);
 9221   effect(KILL cr);
 9222   format %{
 9223     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9224     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9225   %}
 9226   ins_encode %{
 9227     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9228                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9229                /*weak*/ true, noreg);
 9230     __ csetw($res$$Register, Assembler::EQ);
 9231   %}
 9232   ins_pipe(pipe_slow);
 9233 %}
 9234 
 9235 // This pattern is generated automatically from cas.m4.
 9236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9237 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9238 
 9239   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9240   ins_cost(2 * VOLATILE_REF_COST);
 9241   effect(KILL cr);
 9242   format %{
 9243     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9244     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9245   %}
 9246   ins_encode %{
 9247     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9248                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9249                /*weak*/ true, noreg);
 9250     __ csetw($res$$Register, Assembler::EQ);
 9251   %}
 9252   ins_pipe(pipe_slow);
 9253 %}
 9254 
 9255 // This pattern is generated automatically from cas.m4.
 9256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9257 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9258 
 9259   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9260   ins_cost(2 * VOLATILE_REF_COST);
 9261   effect(KILL cr);
 9262   format %{
 9263     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9264     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9265   %}
 9266   ins_encode %{
 9267     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9268                Assembler::word, /*acquire*/ false, /*release*/ true,
 9269                /*weak*/ true, noreg);
 9270     __ csetw($res$$Register, Assembler::EQ);
 9271   %}
 9272   ins_pipe(pipe_slow);
 9273 %}
 9274 
 9275 // This pattern is generated automatically from cas.m4.
 9276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9277 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9278 
 9279   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9280   ins_cost(2 * VOLATILE_REF_COST);
 9281   effect(KILL cr);
 9282   format %{
 9283     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9284     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9285   %}
 9286   ins_encode %{
 9287     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9288                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9289                /*weak*/ true, noreg);
 9290     __ csetw($res$$Register, Assembler::EQ);
 9291   %}
 9292   ins_pipe(pipe_slow);
 9293 %}
 9294 
 9295 // This pattern is generated automatically from cas.m4.
 9296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9297 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9298 
 9299   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9300   ins_cost(2 * VOLATILE_REF_COST);
 9301   effect(KILL cr);
 9302   format %{
 9303     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9304     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9305   %}
 9306   ins_encode %{
 9307     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9308                Assembler::word, /*acquire*/ false, /*release*/ true,
 9309                /*weak*/ true, noreg);
 9310     __ csetw($res$$Register, Assembler::EQ);
 9311   %}
 9312   ins_pipe(pipe_slow);
 9313 %}
 9314 
 9315 // This pattern is generated automatically from cas.m4.
 9316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9317 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9318   predicate(n->as_LoadStore()->barrier_data() == 0);
 9319   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9320   ins_cost(2 * VOLATILE_REF_COST);
 9321   effect(KILL cr);
 9322   format %{
 9323     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9324     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9325   %}
 9326   ins_encode %{
 9327     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9328                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9329                /*weak*/ true, noreg);
 9330     __ csetw($res$$Register, Assembler::EQ);
 9331   %}
 9332   ins_pipe(pipe_slow);
 9333 %}
 9334 
 9335 // This pattern is generated automatically from cas.m4.
 9336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9337 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9338   predicate(needs_acquiring_load_exclusive(n));
 9339   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9340   ins_cost(VOLATILE_REF_COST);
 9341   effect(KILL cr);
 9342   format %{
 9343     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9344     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9345   %}
 9346   ins_encode %{
 9347     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9348                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9349                /*weak*/ true, noreg);
 9350     __ csetw($res$$Register, Assembler::EQ);
 9351   %}
 9352   ins_pipe(pipe_slow);
 9353 %}
 9354 
 9355 // This pattern is generated automatically from cas.m4.
 9356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9357 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9358   predicate(needs_acquiring_load_exclusive(n));
 9359   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9360   ins_cost(VOLATILE_REF_COST);
 9361   effect(KILL cr);
 9362   format %{
 9363     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9364     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9365   %}
 9366   ins_encode %{
 9367     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9368                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9369                /*weak*/ true, noreg);
 9370     __ csetw($res$$Register, Assembler::EQ);
 9371   %}
 9372   ins_pipe(pipe_slow);
 9373 %}
 9374 
 9375 // This pattern is generated automatically from cas.m4.
 9376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9377 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9378   predicate(needs_acquiring_load_exclusive(n));
 9379   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9380   ins_cost(VOLATILE_REF_COST);
 9381   effect(KILL cr);
 9382   format %{
 9383     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9384     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9385   %}
 9386   ins_encode %{
 9387     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9388                Assembler::word, /*acquire*/ true, /*release*/ true,
 9389                /*weak*/ true, noreg);
 9390     __ csetw($res$$Register, Assembler::EQ);
 9391   %}
 9392   ins_pipe(pipe_slow);
 9393 %}
 9394 
 9395 // This pattern is generated automatically from cas.m4.
 9396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9397 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9398   predicate(needs_acquiring_load_exclusive(n));
 9399   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9400   ins_cost(VOLATILE_REF_COST);
 9401   effect(KILL cr);
 9402   format %{
 9403     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9404     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9405   %}
 9406   ins_encode %{
 9407     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9408                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9409                /*weak*/ true, noreg);
 9410     __ csetw($res$$Register, Assembler::EQ);
 9411   %}
 9412   ins_pipe(pipe_slow);
 9413 %}
 9414 
 9415 // This pattern is generated automatically from cas.m4.
 9416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9417 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9418   predicate(needs_acquiring_load_exclusive(n));
 9419   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9420   ins_cost(VOLATILE_REF_COST);
 9421   effect(KILL cr);
 9422   format %{
 9423     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9424     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9425   %}
 9426   ins_encode %{
 9427     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9428                Assembler::word, /*acquire*/ true, /*release*/ true,
 9429                /*weak*/ true, noreg);
 9430     __ csetw($res$$Register, Assembler::EQ);
 9431   %}
 9432   ins_pipe(pipe_slow);
 9433 %}
 9434 
 9435 // This pattern is generated automatically from cas.m4.
 9436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9437 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9438   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9439   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9440   ins_cost(VOLATILE_REF_COST);
 9441   effect(KILL cr);
 9442   format %{
 9443     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9444     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9445   %}
 9446   ins_encode %{
 9447     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9448                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9449                /*weak*/ true, noreg);
 9450     __ csetw($res$$Register, Assembler::EQ);
 9451   %}
 9452   ins_pipe(pipe_slow);
 9453 %}
 9454 
 9455 // END This section of the file is automatically generated. Do not edit --------------
 9456 // ---------------------------------------------------------------------
 9457 
 9458 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9459   match(Set prev (GetAndSetI mem newv));
 9460   ins_cost(2 * VOLATILE_REF_COST);
 9461   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9462   ins_encode %{
 9463     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9464   %}
 9465   ins_pipe(pipe_serial);
 9466 %}
 9467 
 9468 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9469   match(Set prev (GetAndSetL mem newv));
 9470   ins_cost(2 * VOLATILE_REF_COST);
 9471   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9472   ins_encode %{
 9473     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9474   %}
 9475   ins_pipe(pipe_serial);
 9476 %}
 9477 
 9478 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9479   match(Set prev (GetAndSetN mem newv));
 9480   ins_cost(2 * VOLATILE_REF_COST);
 9481   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9482   ins_encode %{
 9483     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9484   %}
 9485   ins_pipe(pipe_serial);
 9486 %}
 9487 
 9488 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9489   predicate(n->as_LoadStore()->barrier_data() == 0);
 9490   match(Set prev (GetAndSetP mem newv));
 9491   ins_cost(2 * VOLATILE_REF_COST);
 9492   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9493   ins_encode %{
 9494     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9495   %}
 9496   ins_pipe(pipe_serial);
 9497 %}
 9498 
 9499 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9500   predicate(needs_acquiring_load_exclusive(n));
 9501   match(Set prev (GetAndSetI mem newv));
 9502   ins_cost(VOLATILE_REF_COST);
 9503   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9504   ins_encode %{
 9505     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9506   %}
 9507   ins_pipe(pipe_serial);
 9508 %}
 9509 
 9510 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9511   predicate(needs_acquiring_load_exclusive(n));
 9512   match(Set prev (GetAndSetL mem newv));
 9513   ins_cost(VOLATILE_REF_COST);
 9514   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9515   ins_encode %{
 9516     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9517   %}
 9518   ins_pipe(pipe_serial);
 9519 %}
 9520 
 9521 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9522   predicate(needs_acquiring_load_exclusive(n));
 9523   match(Set prev (GetAndSetN mem newv));
 9524   ins_cost(VOLATILE_REF_COST);
 9525   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9526   ins_encode %{
 9527     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9528   %}
 9529   ins_pipe(pipe_serial);
 9530 %}
 9531 
 9532 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9533   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9534   match(Set prev (GetAndSetP mem newv));
 9535   ins_cost(VOLATILE_REF_COST);
 9536   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9537   ins_encode %{
 9538     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9539   %}
 9540   ins_pipe(pipe_serial);
 9541 %}
 9542 
 9543 
 9544 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9545   match(Set newval (GetAndAddL mem incr));
 9546   ins_cost(2 * VOLATILE_REF_COST + 1);
 9547   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9548   ins_encode %{
 9549     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9550   %}
 9551   ins_pipe(pipe_serial);
 9552 %}
 9553 
 9554 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9555   predicate(n->as_LoadStore()->result_not_used());
 9556   match(Set dummy (GetAndAddL mem incr));
 9557   ins_cost(2 * VOLATILE_REF_COST);
 9558   format %{ "get_and_addL [$mem], $incr" %}
 9559   ins_encode %{
 9560     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9561   %}
 9562   ins_pipe(pipe_serial);
 9563 %}
 9564 
 9565 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9566   match(Set newval (GetAndAddL mem incr));
 9567   ins_cost(2 * VOLATILE_REF_COST + 1);
 9568   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9569   ins_encode %{
 9570     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9571   %}
 9572   ins_pipe(pipe_serial);
 9573 %}
 9574 
 9575 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9576   predicate(n->as_LoadStore()->result_not_used());
 9577   match(Set dummy (GetAndAddL mem incr));
 9578   ins_cost(2 * VOLATILE_REF_COST);
 9579   format %{ "get_and_addL [$mem], $incr" %}
 9580   ins_encode %{
 9581     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9582   %}
 9583   ins_pipe(pipe_serial);
 9584 %}
 9585 
 9586 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9587   match(Set newval (GetAndAddI mem incr));
 9588   ins_cost(2 * VOLATILE_REF_COST + 1);
 9589   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9590   ins_encode %{
 9591     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9592   %}
 9593   ins_pipe(pipe_serial);
 9594 %}
 9595 
 9596 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9597   predicate(n->as_LoadStore()->result_not_used());
 9598   match(Set dummy (GetAndAddI mem incr));
 9599   ins_cost(2 * VOLATILE_REF_COST);
 9600   format %{ "get_and_addI [$mem], $incr" %}
 9601   ins_encode %{
 9602     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9603   %}
 9604   ins_pipe(pipe_serial);
 9605 %}
 9606 
 9607 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9608   match(Set newval (GetAndAddI mem incr));
 9609   ins_cost(2 * VOLATILE_REF_COST + 1);
 9610   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9611   ins_encode %{
 9612     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9613   %}
 9614   ins_pipe(pipe_serial);
 9615 %}
 9616 
 9617 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9618   predicate(n->as_LoadStore()->result_not_used());
 9619   match(Set dummy (GetAndAddI mem incr));
 9620   ins_cost(2 * VOLATILE_REF_COST);
 9621   format %{ "get_and_addI [$mem], $incr" %}
 9622   ins_encode %{
 9623     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9624   %}
 9625   ins_pipe(pipe_serial);
 9626 %}
 9627 
 9628 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9629   predicate(needs_acquiring_load_exclusive(n));
 9630   match(Set newval (GetAndAddL mem incr));
 9631   ins_cost(VOLATILE_REF_COST + 1);
 9632   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9633   ins_encode %{
 9634     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9635   %}
 9636   ins_pipe(pipe_serial);
 9637 %}
 9638 
 9639 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9640   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9641   match(Set dummy (GetAndAddL mem incr));
 9642   ins_cost(VOLATILE_REF_COST);
 9643   format %{ "get_and_addL_acq [$mem], $incr" %}
 9644   ins_encode %{
 9645     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9646   %}
 9647   ins_pipe(pipe_serial);
 9648 %}
 9649 
 9650 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9651   predicate(needs_acquiring_load_exclusive(n));
 9652   match(Set newval (GetAndAddL mem incr));
 9653   ins_cost(VOLATILE_REF_COST + 1);
 9654   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9655   ins_encode %{
 9656     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9657   %}
 9658   ins_pipe(pipe_serial);
 9659 %}
 9660 
 9661 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9662   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9663   match(Set dummy (GetAndAddL mem incr));
 9664   ins_cost(VOLATILE_REF_COST);
 9665   format %{ "get_and_addL_acq [$mem], $incr" %}
 9666   ins_encode %{
 9667     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9668   %}
 9669   ins_pipe(pipe_serial);
 9670 %}
 9671 
 9672 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9673   predicate(needs_acquiring_load_exclusive(n));
 9674   match(Set newval (GetAndAddI mem incr));
 9675   ins_cost(VOLATILE_REF_COST + 1);
 9676   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9677   ins_encode %{
 9678     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9679   %}
 9680   ins_pipe(pipe_serial);
 9681 %}
 9682 
 9683 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9684   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9685   match(Set dummy (GetAndAddI mem incr));
 9686   ins_cost(VOLATILE_REF_COST);
 9687   format %{ "get_and_addI_acq [$mem], $incr" %}
 9688   ins_encode %{
 9689     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9690   %}
 9691   ins_pipe(pipe_serial);
 9692 %}
 9693 
 9694 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9695   predicate(needs_acquiring_load_exclusive(n));
 9696   match(Set newval (GetAndAddI mem incr));
 9697   ins_cost(VOLATILE_REF_COST + 1);
 9698   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9699   ins_encode %{
 9700     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9701   %}
 9702   ins_pipe(pipe_serial);
 9703 %}
 9704 
 9705 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9706   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9707   match(Set dummy (GetAndAddI mem incr));
 9708   ins_cost(VOLATILE_REF_COST);
 9709   format %{ "get_and_addI_acq [$mem], $incr" %}
 9710   ins_encode %{
 9711     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9712   %}
 9713   ins_pipe(pipe_serial);
 9714 %}
 9715 
 9716 // Manifest a CmpU result in an integer register.
 9717 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9718 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9719 %{
 9720   match(Set dst (CmpU3 src1 src2));
 9721   effect(KILL flags);
 9722 
 9723   ins_cost(INSN_COST * 3);
 9724   format %{
 9725       "cmpw $src1, $src2\n\t"
 9726       "csetw $dst, ne\n\t"
 9727       "cnegw $dst, lo\t# CmpU3(reg)"
 9728   %}
 9729   ins_encode %{
 9730     __ cmpw($src1$$Register, $src2$$Register);
 9731     __ csetw($dst$$Register, Assembler::NE);
 9732     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9733   %}
 9734 
 9735   ins_pipe(pipe_class_default);
 9736 %}
 9737 
 9738 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9739 %{
 9740   match(Set dst (CmpU3 src1 src2));
 9741   effect(KILL flags);
 9742 
 9743   ins_cost(INSN_COST * 3);
 9744   format %{
 9745       "subsw zr, $src1, $src2\n\t"
 9746       "csetw $dst, ne\n\t"
 9747       "cnegw $dst, lo\t# CmpU3(imm)"
 9748   %}
 9749   ins_encode %{
 9750     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9751     __ csetw($dst$$Register, Assembler::NE);
 9752     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9753   %}
 9754 
 9755   ins_pipe(pipe_class_default);
 9756 %}
 9757 
 9758 // Manifest a CmpUL result in an integer register.
 9759 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9760 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9761 %{
 9762   match(Set dst (CmpUL3 src1 src2));
 9763   effect(KILL flags);
 9764 
 9765   ins_cost(INSN_COST * 3);
 9766   format %{
 9767       "cmp $src1, $src2\n\t"
 9768       "csetw $dst, ne\n\t"
 9769       "cnegw $dst, lo\t# CmpUL3(reg)"
 9770   %}
 9771   ins_encode %{
 9772     __ cmp($src1$$Register, $src2$$Register);
 9773     __ csetw($dst$$Register, Assembler::NE);
 9774     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9775   %}
 9776 
 9777   ins_pipe(pipe_class_default);
 9778 %}
 9779 
 9780 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9781 %{
 9782   match(Set dst (CmpUL3 src1 src2));
 9783   effect(KILL flags);
 9784 
 9785   ins_cost(INSN_COST * 3);
 9786   format %{
 9787       "subs zr, $src1, $src2\n\t"
 9788       "csetw $dst, ne\n\t"
 9789       "cnegw $dst, lo\t# CmpUL3(imm)"
 9790   %}
 9791   ins_encode %{
 9792     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9793     __ csetw($dst$$Register, Assembler::NE);
 9794     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9795   %}
 9796 
 9797   ins_pipe(pipe_class_default);
 9798 %}
 9799 
 9800 // Manifest a CmpL result in an integer register.
 9801 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9802 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9803 %{
 9804   match(Set dst (CmpL3 src1 src2));
 9805   effect(KILL flags);
 9806 
 9807   ins_cost(INSN_COST * 3);
 9808   format %{
 9809       "cmp $src1, $src2\n\t"
 9810       "csetw $dst, ne\n\t"
 9811       "cnegw $dst, lt\t# CmpL3(reg)"
 9812   %}
 9813   ins_encode %{
 9814     __ cmp($src1$$Register, $src2$$Register);
 9815     __ csetw($dst$$Register, Assembler::NE);
 9816     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9817   %}
 9818 
 9819   ins_pipe(pipe_class_default);
 9820 %}
 9821 
 9822 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9823 %{
 9824   match(Set dst (CmpL3 src1 src2));
 9825   effect(KILL flags);
 9826 
 9827   ins_cost(INSN_COST * 3);
 9828   format %{
 9829       "subs zr, $src1, $src2\n\t"
 9830       "csetw $dst, ne\n\t"
 9831       "cnegw $dst, lt\t# CmpL3(imm)"
 9832   %}
 9833   ins_encode %{
 9834     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9835     __ csetw($dst$$Register, Assembler::NE);
 9836     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9837   %}
 9838 
 9839   ins_pipe(pipe_class_default);
 9840 %}
 9841 
 9842 // ============================================================================
 9843 // Conditional Move Instructions
 9844 
 9845 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9846 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9847 // define an op class which merged both inputs and use it to type the
 9848 // argument to a single rule. unfortunatelyt his fails because the
 9849 // opclass does not live up to the COND_INTER interface of its
 9850 // component operands. When the generic code tries to negate the
 9851 // operand it ends up running the generci Machoper::negate method
 9852 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9853 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9854 
 9855 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9856   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9857 
 9858   ins_cost(INSN_COST * 2);
 9859   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9860 
 9861   ins_encode %{
 9862     __ cselw(as_Register($dst$$reg),
 9863              as_Register($src2$$reg),
 9864              as_Register($src1$$reg),
 9865              (Assembler::Condition)$cmp$$cmpcode);
 9866   %}
 9867 
 9868   ins_pipe(icond_reg_reg);
 9869 %}
 9870 
 9871 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9872   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9873 
 9874   ins_cost(INSN_COST * 2);
 9875   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9876 
 9877   ins_encode %{
 9878     __ cselw(as_Register($dst$$reg),
 9879              as_Register($src2$$reg),
 9880              as_Register($src1$$reg),
 9881              (Assembler::Condition)$cmp$$cmpcode);
 9882   %}
 9883 
 9884   ins_pipe(icond_reg_reg);
 9885 %}
 9886 
 9887 // special cases where one arg is zero
 9888 
 9889 // n.b. this is selected in preference to the rule above because it
 9890 // avoids loading constant 0 into a source register
 9891 
 9892 // TODO
 9893 // we ought only to be able to cull one of these variants as the ideal
 9894 // transforms ought always to order the zero consistently (to left/right?)
 9895 
 9896 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9897   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9898 
 9899   ins_cost(INSN_COST * 2);
 9900   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9901 
 9902   ins_encode %{
 9903     __ cselw(as_Register($dst$$reg),
 9904              as_Register($src$$reg),
 9905              zr,
 9906              (Assembler::Condition)$cmp$$cmpcode);
 9907   %}
 9908 
 9909   ins_pipe(icond_reg);
 9910 %}
 9911 
 9912 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9913   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9914 
 9915   ins_cost(INSN_COST * 2);
 9916   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9917 
 9918   ins_encode %{
 9919     __ cselw(as_Register($dst$$reg),
 9920              as_Register($src$$reg),
 9921              zr,
 9922              (Assembler::Condition)$cmp$$cmpcode);
 9923   %}
 9924 
 9925   ins_pipe(icond_reg);
 9926 %}
 9927 
 9928 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9929   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9930 
 9931   ins_cost(INSN_COST * 2);
 9932   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9933 
 9934   ins_encode %{
 9935     __ cselw(as_Register($dst$$reg),
 9936              zr,
 9937              as_Register($src$$reg),
 9938              (Assembler::Condition)$cmp$$cmpcode);
 9939   %}
 9940 
 9941   ins_pipe(icond_reg);
 9942 %}
 9943 
 9944 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9945   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9946 
 9947   ins_cost(INSN_COST * 2);
 9948   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9949 
 9950   ins_encode %{
 9951     __ cselw(as_Register($dst$$reg),
 9952              zr,
 9953              as_Register($src$$reg),
 9954              (Assembler::Condition)$cmp$$cmpcode);
 9955   %}
 9956 
 9957   ins_pipe(icond_reg);
 9958 %}
 9959 
 9960 // special case for creating a boolean 0 or 1
 9961 
 9962 // n.b. this is selected in preference to the rule above because it
 9963 // avoids loading constants 0 and 1 into a source register
 9964 
 9965 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9966   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9967 
 9968   ins_cost(INSN_COST * 2);
 9969   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9970 
 9971   ins_encode %{
 9972     // equivalently
 9973     // cset(as_Register($dst$$reg),
 9974     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9975     __ csincw(as_Register($dst$$reg),
 9976              zr,
 9977              zr,
 9978              (Assembler::Condition)$cmp$$cmpcode);
 9979   %}
 9980 
 9981   ins_pipe(icond_none);
 9982 %}
 9983 
 9984 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9985   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9986 
 9987   ins_cost(INSN_COST * 2);
 9988   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9989 
 9990   ins_encode %{
 9991     // equivalently
 9992     // cset(as_Register($dst$$reg),
 9993     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9994     __ csincw(as_Register($dst$$reg),
 9995              zr,
 9996              zr,
 9997              (Assembler::Condition)$cmp$$cmpcode);
 9998   %}
 9999 
10000   ins_pipe(icond_none);
10001 %}
10002 
10003 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10004   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10005 
10006   ins_cost(INSN_COST * 2);
10007   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10008 
10009   ins_encode %{
10010     __ csel(as_Register($dst$$reg),
10011             as_Register($src2$$reg),
10012             as_Register($src1$$reg),
10013             (Assembler::Condition)$cmp$$cmpcode);
10014   %}
10015 
10016   ins_pipe(icond_reg_reg);
10017 %}
10018 
10019 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10020   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10021 
10022   ins_cost(INSN_COST * 2);
10023   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10024 
10025   ins_encode %{
10026     __ csel(as_Register($dst$$reg),
10027             as_Register($src2$$reg),
10028             as_Register($src1$$reg),
10029             (Assembler::Condition)$cmp$$cmpcode);
10030   %}
10031 
10032   ins_pipe(icond_reg_reg);
10033 %}
10034 
10035 // special cases where one arg is zero
10036 
10037 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10038   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10039 
10040   ins_cost(INSN_COST * 2);
10041   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10042 
10043   ins_encode %{
10044     __ csel(as_Register($dst$$reg),
10045             zr,
10046             as_Register($src$$reg),
10047             (Assembler::Condition)$cmp$$cmpcode);
10048   %}
10049 
10050   ins_pipe(icond_reg);
10051 %}
10052 
10053 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10054   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10055 
10056   ins_cost(INSN_COST * 2);
10057   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10058 
10059   ins_encode %{
10060     __ csel(as_Register($dst$$reg),
10061             zr,
10062             as_Register($src$$reg),
10063             (Assembler::Condition)$cmp$$cmpcode);
10064   %}
10065 
10066   ins_pipe(icond_reg);
10067 %}
10068 
10069 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10070   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10071 
10072   ins_cost(INSN_COST * 2);
10073   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10074 
10075   ins_encode %{
10076     __ csel(as_Register($dst$$reg),
10077             as_Register($src$$reg),
10078             zr,
10079             (Assembler::Condition)$cmp$$cmpcode);
10080   %}
10081 
10082   ins_pipe(icond_reg);
10083 %}
10084 
10085 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10086   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10087 
10088   ins_cost(INSN_COST * 2);
10089   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10090 
10091   ins_encode %{
10092     __ csel(as_Register($dst$$reg),
10093             as_Register($src$$reg),
10094             zr,
10095             (Assembler::Condition)$cmp$$cmpcode);
10096   %}
10097 
10098   ins_pipe(icond_reg);
10099 %}
10100 
10101 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10102   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10103 
10104   ins_cost(INSN_COST * 2);
10105   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10106 
10107   ins_encode %{
10108     __ csel(as_Register($dst$$reg),
10109             as_Register($src2$$reg),
10110             as_Register($src1$$reg),
10111             (Assembler::Condition)$cmp$$cmpcode);
10112   %}
10113 
10114   ins_pipe(icond_reg_reg);
10115 %}
10116 
10117 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10118   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10119 
10120   ins_cost(INSN_COST * 2);
10121   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10122 
10123   ins_encode %{
10124     __ csel(as_Register($dst$$reg),
10125             as_Register($src2$$reg),
10126             as_Register($src1$$reg),
10127             (Assembler::Condition)$cmp$$cmpcode);
10128   %}
10129 
10130   ins_pipe(icond_reg_reg);
10131 %}
10132 
10133 // special cases where one arg is zero
10134 
10135 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10136   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10137 
10138   ins_cost(INSN_COST * 2);
10139   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10140 
10141   ins_encode %{
10142     __ csel(as_Register($dst$$reg),
10143             zr,
10144             as_Register($src$$reg),
10145             (Assembler::Condition)$cmp$$cmpcode);
10146   %}
10147 
10148   ins_pipe(icond_reg);
10149 %}
10150 
10151 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10152   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10153 
10154   ins_cost(INSN_COST * 2);
10155   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10156 
10157   ins_encode %{
10158     __ csel(as_Register($dst$$reg),
10159             zr,
10160             as_Register($src$$reg),
10161             (Assembler::Condition)$cmp$$cmpcode);
10162   %}
10163 
10164   ins_pipe(icond_reg);
10165 %}
10166 
10167 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10168   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10169 
10170   ins_cost(INSN_COST * 2);
10171   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10172 
10173   ins_encode %{
10174     __ csel(as_Register($dst$$reg),
10175             as_Register($src$$reg),
10176             zr,
10177             (Assembler::Condition)$cmp$$cmpcode);
10178   %}
10179 
10180   ins_pipe(icond_reg);
10181 %}
10182 
10183 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10184   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10185 
10186   ins_cost(INSN_COST * 2);
10187   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10188 
10189   ins_encode %{
10190     __ csel(as_Register($dst$$reg),
10191             as_Register($src$$reg),
10192             zr,
10193             (Assembler::Condition)$cmp$$cmpcode);
10194   %}
10195 
10196   ins_pipe(icond_reg);
10197 %}
10198 
10199 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10200   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10201 
10202   ins_cost(INSN_COST * 2);
10203   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10204 
10205   ins_encode %{
10206     __ cselw(as_Register($dst$$reg),
10207              as_Register($src2$$reg),
10208              as_Register($src1$$reg),
10209              (Assembler::Condition)$cmp$$cmpcode);
10210   %}
10211 
10212   ins_pipe(icond_reg_reg);
10213 %}
10214 
10215 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10216   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10217 
10218   ins_cost(INSN_COST * 2);
10219   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10220 
10221   ins_encode %{
10222     __ cselw(as_Register($dst$$reg),
10223              as_Register($src2$$reg),
10224              as_Register($src1$$reg),
10225              (Assembler::Condition)$cmp$$cmpcode);
10226   %}
10227 
10228   ins_pipe(icond_reg_reg);
10229 %}
10230 
10231 // special cases where one arg is zero
10232 
10233 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10234   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10235 
10236   ins_cost(INSN_COST * 2);
10237   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10238 
10239   ins_encode %{
10240     __ cselw(as_Register($dst$$reg),
10241              zr,
10242              as_Register($src$$reg),
10243              (Assembler::Condition)$cmp$$cmpcode);
10244   %}
10245 
10246   ins_pipe(icond_reg);
10247 %}
10248 
10249 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10250   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10251 
10252   ins_cost(INSN_COST * 2);
10253   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10254 
10255   ins_encode %{
10256     __ cselw(as_Register($dst$$reg),
10257              zr,
10258              as_Register($src$$reg),
10259              (Assembler::Condition)$cmp$$cmpcode);
10260   %}
10261 
10262   ins_pipe(icond_reg);
10263 %}
10264 
10265 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10266   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10267 
10268   ins_cost(INSN_COST * 2);
10269   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10270 
10271   ins_encode %{
10272     __ cselw(as_Register($dst$$reg),
10273              as_Register($src$$reg),
10274              zr,
10275              (Assembler::Condition)$cmp$$cmpcode);
10276   %}
10277 
10278   ins_pipe(icond_reg);
10279 %}
10280 
10281 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10282   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10283 
10284   ins_cost(INSN_COST * 2);
10285   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10286 
10287   ins_encode %{
10288     __ cselw(as_Register($dst$$reg),
10289              as_Register($src$$reg),
10290              zr,
10291              (Assembler::Condition)$cmp$$cmpcode);
10292   %}
10293 
10294   ins_pipe(icond_reg);
10295 %}
10296 
10297 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10298 %{
10299   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10300 
10301   ins_cost(INSN_COST * 3);
10302 
10303   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10304   ins_encode %{
10305     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10306     __ fcsels(as_FloatRegister($dst$$reg),
10307               as_FloatRegister($src2$$reg),
10308               as_FloatRegister($src1$$reg),
10309               cond);
10310   %}
10311 
10312   ins_pipe(fp_cond_reg_reg_s);
10313 %}
10314 
10315 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10316 %{
10317   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10318 
10319   ins_cost(INSN_COST * 3);
10320 
10321   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10322   ins_encode %{
10323     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10324     __ fcsels(as_FloatRegister($dst$$reg),
10325               as_FloatRegister($src2$$reg),
10326               as_FloatRegister($src1$$reg),
10327               cond);
10328   %}
10329 
10330   ins_pipe(fp_cond_reg_reg_s);
10331 %}
10332 
10333 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10334 %{
10335   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10336 
10337   ins_cost(INSN_COST * 3);
10338 
10339   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10340   ins_encode %{
10341     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10342     __ fcseld(as_FloatRegister($dst$$reg),
10343               as_FloatRegister($src2$$reg),
10344               as_FloatRegister($src1$$reg),
10345               cond);
10346   %}
10347 
10348   ins_pipe(fp_cond_reg_reg_d);
10349 %}
10350 
10351 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10352 %{
10353   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10354 
10355   ins_cost(INSN_COST * 3);
10356 
10357   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10358   ins_encode %{
10359     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10360     __ fcseld(as_FloatRegister($dst$$reg),
10361               as_FloatRegister($src2$$reg),
10362               as_FloatRegister($src1$$reg),
10363               cond);
10364   %}
10365 
10366   ins_pipe(fp_cond_reg_reg_d);
10367 %}
10368 
10369 // ============================================================================
10370 // Arithmetic Instructions
10371 //
10372 
10373 // Integer Addition
10374 
10375 // TODO
10376 // these currently employ operations which do not set CR and hence are
10377 // not flagged as killing CR but we would like to isolate the cases
10378 // where we want to set flags from those where we don't. need to work
10379 // out how to do that.
10380 
10381 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10382   match(Set dst (AddI src1 src2));
10383 
10384   ins_cost(INSN_COST);
10385   format %{ "addw  $dst, $src1, $src2" %}
10386 
10387   ins_encode %{
10388     __ addw(as_Register($dst$$reg),
10389             as_Register($src1$$reg),
10390             as_Register($src2$$reg));
10391   %}
10392 
10393   ins_pipe(ialu_reg_reg);
10394 %}
10395 
10396 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10397   match(Set dst (AddI src1 src2));
10398 
10399   ins_cost(INSN_COST);
10400   format %{ "addw $dst, $src1, $src2" %}
10401 
10402   // use opcode to indicate that this is an add not a sub
10403   opcode(0x0);
10404 
10405   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10406 
10407   ins_pipe(ialu_reg_imm);
10408 %}
10409 
10410 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10411   match(Set dst (AddI (ConvL2I src1) src2));
10412 
10413   ins_cost(INSN_COST);
10414   format %{ "addw $dst, $src1, $src2" %}
10415 
10416   // use opcode to indicate that this is an add not a sub
10417   opcode(0x0);
10418 
10419   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10420 
10421   ins_pipe(ialu_reg_imm);
10422 %}
10423 
10424 // Pointer Addition
10425 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10426   match(Set dst (AddP src1 src2));
10427 
10428   ins_cost(INSN_COST);
10429   format %{ "add $dst, $src1, $src2\t# ptr" %}
10430 
10431   ins_encode %{
10432     __ add(as_Register($dst$$reg),
10433            as_Register($src1$$reg),
10434            as_Register($src2$$reg));
10435   %}
10436 
10437   ins_pipe(ialu_reg_reg);
10438 %}
10439 
10440 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10441   match(Set dst (AddP src1 (ConvI2L src2)));
10442 
10443   ins_cost(1.9 * INSN_COST);
10444   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10445 
10446   ins_encode %{
10447     __ add(as_Register($dst$$reg),
10448            as_Register($src1$$reg),
10449            as_Register($src2$$reg), ext::sxtw);
10450   %}
10451 
10452   ins_pipe(ialu_reg_reg);
10453 %}
10454 
10455 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10456   match(Set dst (AddP src1 (LShiftL src2 scale)));
10457 
10458   ins_cost(1.9 * INSN_COST);
10459   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10460 
10461   ins_encode %{
10462     __ lea(as_Register($dst$$reg),
10463            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10464                    Address::lsl($scale$$constant)));
10465   %}
10466 
10467   ins_pipe(ialu_reg_reg_shift);
10468 %}
10469 
10470 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10471   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10472 
10473   ins_cost(1.9 * INSN_COST);
10474   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10475 
10476   ins_encode %{
10477     __ lea(as_Register($dst$$reg),
10478            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10479                    Address::sxtw($scale$$constant)));
10480   %}
10481 
10482   ins_pipe(ialu_reg_reg_shift);
10483 %}
10484 
10485 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10486   match(Set dst (LShiftL (ConvI2L src) scale));
10487 
10488   ins_cost(INSN_COST);
10489   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10490 
10491   ins_encode %{
10492     __ sbfiz(as_Register($dst$$reg),
10493           as_Register($src$$reg),
10494           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10495   %}
10496 
10497   ins_pipe(ialu_reg_shift);
10498 %}
10499 
10500 // Pointer Immediate Addition
10501 // n.b. this needs to be more expensive than using an indirect memory
10502 // operand
10503 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10504   match(Set dst (AddP src1 src2));
10505 
10506   ins_cost(INSN_COST);
10507   format %{ "add $dst, $src1, $src2\t# ptr" %}
10508 
10509   // use opcode to indicate that this is an add not a sub
10510   opcode(0x0);
10511 
10512   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10513 
10514   ins_pipe(ialu_reg_imm);
10515 %}
10516 
10517 // Long Addition
10518 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10519 
10520   match(Set dst (AddL src1 src2));
10521 
10522   ins_cost(INSN_COST);
10523   format %{ "add  $dst, $src1, $src2" %}
10524 
10525   ins_encode %{
10526     __ add(as_Register($dst$$reg),
10527            as_Register($src1$$reg),
10528            as_Register($src2$$reg));
10529   %}
10530 
10531   ins_pipe(ialu_reg_reg);
10532 %}
10533 
10534 // No constant pool entries requiredLong Immediate Addition.
10535 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10536   match(Set dst (AddL src1 src2));
10537 
10538   ins_cost(INSN_COST);
10539   format %{ "add $dst, $src1, $src2" %}
10540 
10541   // use opcode to indicate that this is an add not a sub
10542   opcode(0x0);
10543 
10544   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10545 
10546   ins_pipe(ialu_reg_imm);
10547 %}
10548 
10549 // Integer Subtraction
10550 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10551   match(Set dst (SubI src1 src2));
10552 
10553   ins_cost(INSN_COST);
10554   format %{ "subw  $dst, $src1, $src2" %}
10555 
10556   ins_encode %{
10557     __ subw(as_Register($dst$$reg),
10558             as_Register($src1$$reg),
10559             as_Register($src2$$reg));
10560   %}
10561 
10562   ins_pipe(ialu_reg_reg);
10563 %}
10564 
10565 // Immediate Subtraction
10566 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10567   match(Set dst (SubI src1 src2));
10568 
10569   ins_cost(INSN_COST);
10570   format %{ "subw $dst, $src1, $src2" %}
10571 
10572   // use opcode to indicate that this is a sub not an add
10573   opcode(0x1);
10574 
10575   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10576 
10577   ins_pipe(ialu_reg_imm);
10578 %}
10579 
10580 // Long Subtraction
10581 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10582 
10583   match(Set dst (SubL src1 src2));
10584 
10585   ins_cost(INSN_COST);
10586   format %{ "sub  $dst, $src1, $src2" %}
10587 
10588   ins_encode %{
10589     __ sub(as_Register($dst$$reg),
10590            as_Register($src1$$reg),
10591            as_Register($src2$$reg));
10592   %}
10593 
10594   ins_pipe(ialu_reg_reg);
10595 %}
10596 
10597 // No constant pool entries requiredLong Immediate Subtraction.
10598 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10599   match(Set dst (SubL src1 src2));
10600 
10601   ins_cost(INSN_COST);
10602   format %{ "sub$dst, $src1, $src2" %}
10603 
10604   // use opcode to indicate that this is a sub not an add
10605   opcode(0x1);
10606 
10607   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10608 
10609   ins_pipe(ialu_reg_imm);
10610 %}
10611 
10612 // Integer Negation (special case for sub)
10613 
10614 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10615   match(Set dst (SubI zero src));
10616 
10617   ins_cost(INSN_COST);
10618   format %{ "negw $dst, $src\t# int" %}
10619 
10620   ins_encode %{
10621     __ negw(as_Register($dst$$reg),
10622             as_Register($src$$reg));
10623   %}
10624 
10625   ins_pipe(ialu_reg);
10626 %}
10627 
10628 // Long Negation
10629 
10630 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10631   match(Set dst (SubL zero src));
10632 
10633   ins_cost(INSN_COST);
10634   format %{ "neg $dst, $src\t# long" %}
10635 
10636   ins_encode %{
10637     __ neg(as_Register($dst$$reg),
10638            as_Register($src$$reg));
10639   %}
10640 
10641   ins_pipe(ialu_reg);
10642 %}
10643 
10644 // Integer Multiply
10645 
10646 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10647   match(Set dst (MulI src1 src2));
10648 
10649   ins_cost(INSN_COST * 3);
10650   format %{ "mulw  $dst, $src1, $src2" %}
10651 
10652   ins_encode %{
10653     __ mulw(as_Register($dst$$reg),
10654             as_Register($src1$$reg),
10655             as_Register($src2$$reg));
10656   %}
10657 
10658   ins_pipe(imul_reg_reg);
10659 %}
10660 
10661 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10662   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10663 
10664   ins_cost(INSN_COST * 3);
10665   format %{ "smull  $dst, $src1, $src2" %}
10666 
10667   ins_encode %{
10668     __ smull(as_Register($dst$$reg),
10669              as_Register($src1$$reg),
10670              as_Register($src2$$reg));
10671   %}
10672 
10673   ins_pipe(imul_reg_reg);
10674 %}
10675 
10676 // Long Multiply
10677 
10678 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10679   match(Set dst (MulL src1 src2));
10680 
10681   ins_cost(INSN_COST * 5);
10682   format %{ "mul  $dst, $src1, $src2" %}
10683 
10684   ins_encode %{
10685     __ mul(as_Register($dst$$reg),
10686            as_Register($src1$$reg),
10687            as_Register($src2$$reg));
10688   %}
10689 
10690   ins_pipe(lmul_reg_reg);
10691 %}
10692 
10693 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10694 %{
10695   match(Set dst (MulHiL src1 src2));
10696 
10697   ins_cost(INSN_COST * 7);
10698   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10699 
10700   ins_encode %{
10701     __ smulh(as_Register($dst$$reg),
10702              as_Register($src1$$reg),
10703              as_Register($src2$$reg));
10704   %}
10705 
10706   ins_pipe(lmul_reg_reg);
10707 %}
10708 
10709 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10710 %{
10711   match(Set dst (UMulHiL src1 src2));
10712 
10713   ins_cost(INSN_COST * 7);
10714   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10715 
10716   ins_encode %{
10717     __ umulh(as_Register($dst$$reg),
10718              as_Register($src1$$reg),
10719              as_Register($src2$$reg));
10720   %}
10721 
10722   ins_pipe(lmul_reg_reg);
10723 %}
10724 
10725 // Combined Integer Multiply & Add/Sub
10726 
10727 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10728   match(Set dst (AddI src3 (MulI src1 src2)));
10729 
10730   ins_cost(INSN_COST * 3);
10731   format %{ "madd  $dst, $src1, $src2, $src3" %}
10732 
10733   ins_encode %{
10734     __ maddw(as_Register($dst$$reg),
10735              as_Register($src1$$reg),
10736              as_Register($src2$$reg),
10737              as_Register($src3$$reg));
10738   %}
10739 
10740   ins_pipe(imac_reg_reg);
10741 %}
10742 
10743 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10744   match(Set dst (SubI src3 (MulI src1 src2)));
10745 
10746   ins_cost(INSN_COST * 3);
10747   format %{ "msub  $dst, $src1, $src2, $src3" %}
10748 
10749   ins_encode %{
10750     __ msubw(as_Register($dst$$reg),
10751              as_Register($src1$$reg),
10752              as_Register($src2$$reg),
10753              as_Register($src3$$reg));
10754   %}
10755 
10756   ins_pipe(imac_reg_reg);
10757 %}
10758 
10759 // Combined Integer Multiply & Neg
10760 
10761 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10762   match(Set dst (MulI (SubI zero src1) src2));
10763 
10764   ins_cost(INSN_COST * 3);
10765   format %{ "mneg  $dst, $src1, $src2" %}
10766 
10767   ins_encode %{
10768     __ mnegw(as_Register($dst$$reg),
10769              as_Register($src1$$reg),
10770              as_Register($src2$$reg));
10771   %}
10772 
10773   ins_pipe(imac_reg_reg);
10774 %}
10775 
10776 // Combined Long Multiply & Add/Sub
10777 
10778 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10779   match(Set dst (AddL src3 (MulL src1 src2)));
10780 
10781   ins_cost(INSN_COST * 5);
10782   format %{ "madd  $dst, $src1, $src2, $src3" %}
10783 
10784   ins_encode %{
10785     __ madd(as_Register($dst$$reg),
10786             as_Register($src1$$reg),
10787             as_Register($src2$$reg),
10788             as_Register($src3$$reg));
10789   %}
10790 
10791   ins_pipe(lmac_reg_reg);
10792 %}
10793 
10794 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10795   match(Set dst (SubL src3 (MulL src1 src2)));
10796 
10797   ins_cost(INSN_COST * 5);
10798   format %{ "msub  $dst, $src1, $src2, $src3" %}
10799 
10800   ins_encode %{
10801     __ msub(as_Register($dst$$reg),
10802             as_Register($src1$$reg),
10803             as_Register($src2$$reg),
10804             as_Register($src3$$reg));
10805   %}
10806 
10807   ins_pipe(lmac_reg_reg);
10808 %}
10809 
10810 // Combined Long Multiply & Neg
10811 
10812 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10813   match(Set dst (MulL (SubL zero src1) src2));
10814 
10815   ins_cost(INSN_COST * 5);
10816   format %{ "mneg  $dst, $src1, $src2" %}
10817 
10818   ins_encode %{
10819     __ mneg(as_Register($dst$$reg),
10820             as_Register($src1$$reg),
10821             as_Register($src2$$reg));
10822   %}
10823 
10824   ins_pipe(lmac_reg_reg);
10825 %}
10826 
10827 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10828 
10829 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10830   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10831 
10832   ins_cost(INSN_COST * 3);
10833   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10834 
10835   ins_encode %{
10836     __ smaddl(as_Register($dst$$reg),
10837               as_Register($src1$$reg),
10838               as_Register($src2$$reg),
10839               as_Register($src3$$reg));
10840   %}
10841 
10842   ins_pipe(imac_reg_reg);
10843 %}
10844 
10845 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10846   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10847 
10848   ins_cost(INSN_COST * 3);
10849   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10850 
10851   ins_encode %{
10852     __ smsubl(as_Register($dst$$reg),
10853               as_Register($src1$$reg),
10854               as_Register($src2$$reg),
10855               as_Register($src3$$reg));
10856   %}
10857 
10858   ins_pipe(imac_reg_reg);
10859 %}
10860 
10861 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10862   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10863 
10864   ins_cost(INSN_COST * 3);
10865   format %{ "smnegl  $dst, $src1, $src2" %}
10866 
10867   ins_encode %{
10868     __ smnegl(as_Register($dst$$reg),
10869               as_Register($src1$$reg),
10870               as_Register($src2$$reg));
10871   %}
10872 
10873   ins_pipe(imac_reg_reg);
10874 %}
10875 
10876 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10877 
10878 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10879   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10880 
10881   ins_cost(INSN_COST * 5);
10882   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10883             "maddw $dst, $src3, $src4, rscratch1" %}
10884 
10885   ins_encode %{
10886     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10887     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10888 
10889   ins_pipe(imac_reg_reg);
10890 %}
10891 
10892 // Integer Divide
10893 
10894 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10895   match(Set dst (DivI src1 src2));
10896 
10897   ins_cost(INSN_COST * 19);
10898   format %{ "sdivw  $dst, $src1, $src2" %}
10899 
10900   ins_encode(aarch64_enc_divw(dst, src1, src2));
10901   ins_pipe(idiv_reg_reg);
10902 %}
10903 
10904 // Long Divide
10905 
10906 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10907   match(Set dst (DivL src1 src2));
10908 
10909   ins_cost(INSN_COST * 35);
10910   format %{ "sdiv   $dst, $src1, $src2" %}
10911 
10912   ins_encode(aarch64_enc_div(dst, src1, src2));
10913   ins_pipe(ldiv_reg_reg);
10914 %}
10915 
10916 // Integer Remainder
10917 
10918 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10919   match(Set dst (ModI src1 src2));
10920 
10921   ins_cost(INSN_COST * 22);
10922   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10923             "msubw  $dst, rscratch1, $src2, $src1" %}
10924 
10925   ins_encode(aarch64_enc_modw(dst, src1, src2));
10926   ins_pipe(idiv_reg_reg);
10927 %}
10928 
10929 // Long Remainder
10930 
10931 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10932   match(Set dst (ModL src1 src2));
10933 
10934   ins_cost(INSN_COST * 38);
10935   format %{ "sdiv   rscratch1, $src1, $src2\n"
10936             "msub   $dst, rscratch1, $src2, $src1" %}
10937 
10938   ins_encode(aarch64_enc_mod(dst, src1, src2));
10939   ins_pipe(ldiv_reg_reg);
10940 %}
10941 
10942 // Unsigned Integer Divide
10943 
10944 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10945   match(Set dst (UDivI src1 src2));
10946 
10947   ins_cost(INSN_COST * 19);
10948   format %{ "udivw  $dst, $src1, $src2" %}
10949 
10950   ins_encode %{
10951     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10952   %}
10953 
10954   ins_pipe(idiv_reg_reg);
10955 %}
10956 
10957 //  Unsigned Long Divide
10958 
10959 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10960   match(Set dst (UDivL src1 src2));
10961 
10962   ins_cost(INSN_COST * 35);
10963   format %{ "udiv   $dst, $src1, $src2" %}
10964 
10965   ins_encode %{
10966     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10967   %}
10968 
10969   ins_pipe(ldiv_reg_reg);
10970 %}
10971 
10972 // Unsigned Integer Remainder
10973 
10974 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10975   match(Set dst (UModI src1 src2));
10976 
10977   ins_cost(INSN_COST * 22);
10978   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10979             "msubw  $dst, rscratch1, $src2, $src1" %}
10980 
10981   ins_encode %{
10982     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10983     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10984   %}
10985 
10986   ins_pipe(idiv_reg_reg);
10987 %}
10988 
10989 // Unsigned Long Remainder
10990 
10991 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10992   match(Set dst (UModL src1 src2));
10993 
10994   ins_cost(INSN_COST * 38);
10995   format %{ "udiv   rscratch1, $src1, $src2\n"
10996             "msub   $dst, rscratch1, $src2, $src1" %}
10997 
10998   ins_encode %{
10999     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11000     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11001   %}
11002 
11003   ins_pipe(ldiv_reg_reg);
11004 %}
11005 
11006 // Integer Shifts
11007 
11008 // Shift Left Register
11009 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11010   match(Set dst (LShiftI src1 src2));
11011 
11012   ins_cost(INSN_COST * 2);
11013   format %{ "lslvw  $dst, $src1, $src2" %}
11014 
11015   ins_encode %{
11016     __ lslvw(as_Register($dst$$reg),
11017              as_Register($src1$$reg),
11018              as_Register($src2$$reg));
11019   %}
11020 
11021   ins_pipe(ialu_reg_reg_vshift);
11022 %}
11023 
11024 // Shift Left Immediate
11025 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11026   match(Set dst (LShiftI src1 src2));
11027 
11028   ins_cost(INSN_COST);
11029   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11030 
11031   ins_encode %{
11032     __ lslw(as_Register($dst$$reg),
11033             as_Register($src1$$reg),
11034             $src2$$constant & 0x1f);
11035   %}
11036 
11037   ins_pipe(ialu_reg_shift);
11038 %}
11039 
11040 // Shift Right Logical Register
11041 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11042   match(Set dst (URShiftI src1 src2));
11043 
11044   ins_cost(INSN_COST * 2);
11045   format %{ "lsrvw  $dst, $src1, $src2" %}
11046 
11047   ins_encode %{
11048     __ lsrvw(as_Register($dst$$reg),
11049              as_Register($src1$$reg),
11050              as_Register($src2$$reg));
11051   %}
11052 
11053   ins_pipe(ialu_reg_reg_vshift);
11054 %}
11055 
11056 // Shift Right Logical Immediate
11057 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11058   match(Set dst (URShiftI src1 src2));
11059 
11060   ins_cost(INSN_COST);
11061   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11062 
11063   ins_encode %{
11064     __ lsrw(as_Register($dst$$reg),
11065             as_Register($src1$$reg),
11066             $src2$$constant & 0x1f);
11067   %}
11068 
11069   ins_pipe(ialu_reg_shift);
11070 %}
11071 
11072 // Shift Right Arithmetic Register
11073 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11074   match(Set dst (RShiftI src1 src2));
11075 
11076   ins_cost(INSN_COST * 2);
11077   format %{ "asrvw  $dst, $src1, $src2" %}
11078 
11079   ins_encode %{
11080     __ asrvw(as_Register($dst$$reg),
11081              as_Register($src1$$reg),
11082              as_Register($src2$$reg));
11083   %}
11084 
11085   ins_pipe(ialu_reg_reg_vshift);
11086 %}
11087 
11088 // Shift Right Arithmetic Immediate
11089 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11090   match(Set dst (RShiftI src1 src2));
11091 
11092   ins_cost(INSN_COST);
11093   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11094 
11095   ins_encode %{
11096     __ asrw(as_Register($dst$$reg),
11097             as_Register($src1$$reg),
11098             $src2$$constant & 0x1f);
11099   %}
11100 
11101   ins_pipe(ialu_reg_shift);
11102 %}
11103 
11104 // Combined Int Mask and Right Shift (using UBFM)
11105 // TODO
11106 
11107 // Long Shifts
11108 
11109 // Shift Left Register
11110 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11111   match(Set dst (LShiftL src1 src2));
11112 
11113   ins_cost(INSN_COST * 2);
11114   format %{ "lslv  $dst, $src1, $src2" %}
11115 
11116   ins_encode %{
11117     __ lslv(as_Register($dst$$reg),
11118             as_Register($src1$$reg),
11119             as_Register($src2$$reg));
11120   %}
11121 
11122   ins_pipe(ialu_reg_reg_vshift);
11123 %}
11124 
11125 // Shift Left Immediate
11126 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11127   match(Set dst (LShiftL src1 src2));
11128 
11129   ins_cost(INSN_COST);
11130   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11131 
11132   ins_encode %{
11133     __ lsl(as_Register($dst$$reg),
11134             as_Register($src1$$reg),
11135             $src2$$constant & 0x3f);
11136   %}
11137 
11138   ins_pipe(ialu_reg_shift);
11139 %}
11140 
11141 // Shift Right Logical Register
11142 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11143   match(Set dst (URShiftL src1 src2));
11144 
11145   ins_cost(INSN_COST * 2);
11146   format %{ "lsrv  $dst, $src1, $src2" %}
11147 
11148   ins_encode %{
11149     __ lsrv(as_Register($dst$$reg),
11150             as_Register($src1$$reg),
11151             as_Register($src2$$reg));
11152   %}
11153 
11154   ins_pipe(ialu_reg_reg_vshift);
11155 %}
11156 
11157 // Shift Right Logical Immediate
11158 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11159   match(Set dst (URShiftL src1 src2));
11160 
11161   ins_cost(INSN_COST);
11162   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11163 
11164   ins_encode %{
11165     __ lsr(as_Register($dst$$reg),
11166            as_Register($src1$$reg),
11167            $src2$$constant & 0x3f);
11168   %}
11169 
11170   ins_pipe(ialu_reg_shift);
11171 %}
11172 
11173 // A special-case pattern for card table stores.
11174 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11175   match(Set dst (URShiftL (CastP2X src1) src2));
11176 
11177   ins_cost(INSN_COST);
11178   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11179 
11180   ins_encode %{
11181     __ lsr(as_Register($dst$$reg),
11182            as_Register($src1$$reg),
11183            $src2$$constant & 0x3f);
11184   %}
11185 
11186   ins_pipe(ialu_reg_shift);
11187 %}
11188 
11189 // Shift Right Arithmetic Register
11190 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11191   match(Set dst (RShiftL src1 src2));
11192 
11193   ins_cost(INSN_COST * 2);
11194   format %{ "asrv  $dst, $src1, $src2" %}
11195 
11196   ins_encode %{
11197     __ asrv(as_Register($dst$$reg),
11198             as_Register($src1$$reg),
11199             as_Register($src2$$reg));
11200   %}
11201 
11202   ins_pipe(ialu_reg_reg_vshift);
11203 %}
11204 
11205 // Shift Right Arithmetic Immediate
11206 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11207   match(Set dst (RShiftL src1 src2));
11208 
11209   ins_cost(INSN_COST);
11210   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11211 
11212   ins_encode %{
11213     __ asr(as_Register($dst$$reg),
11214            as_Register($src1$$reg),
11215            $src2$$constant & 0x3f);
11216   %}
11217 
11218   ins_pipe(ialu_reg_shift);
11219 %}
11220 
11221 // BEGIN This section of the file is automatically generated. Do not edit --------------
11222 // This section is generated from aarch64_ad.m4
11223 
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct regL_not_reg(iRegLNoSp dst,
11227                          iRegL src1, immL_M1 m1,
11228                          rFlagsReg cr) %{
11229   match(Set dst (XorL src1 m1));
11230   ins_cost(INSN_COST);
11231   format %{ "eon  $dst, $src1, zr" %}
11232 
11233   ins_encode %{
11234     __ eon(as_Register($dst$$reg),
11235               as_Register($src1$$reg),
11236               zr,
11237               Assembler::LSL, 0);
11238   %}
11239 
11240   ins_pipe(ialu_reg);
11241 %}
11242 
11243 // This pattern is automatically generated from aarch64_ad.m4
11244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11245 instruct regI_not_reg(iRegINoSp dst,
11246                          iRegIorL2I src1, immI_M1 m1,
11247                          rFlagsReg cr) %{
11248   match(Set dst (XorI src1 m1));
11249   ins_cost(INSN_COST);
11250   format %{ "eonw  $dst, $src1, zr" %}
11251 
11252   ins_encode %{
11253     __ eonw(as_Register($dst$$reg),
11254               as_Register($src1$$reg),
11255               zr,
11256               Assembler::LSL, 0);
11257   %}
11258 
11259   ins_pipe(ialu_reg);
11260 %}
11261 
11262 // This pattern is automatically generated from aarch64_ad.m4
11263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11264 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11265                               immI0 zero, iRegIorL2I src1, immI src2) %{
11266   match(Set dst (SubI zero (URShiftI src1 src2)));
11267 
11268   ins_cost(1.9 * INSN_COST);
11269   format %{ "negw  $dst, $src1, LSR $src2" %}
11270 
11271   ins_encode %{
11272     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11273             Assembler::LSR, $src2$$constant & 0x1f);
11274   %}
11275 
11276   ins_pipe(ialu_reg_shift);
11277 %}
11278 
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11282                               immI0 zero, iRegIorL2I src1, immI src2) %{
11283   match(Set dst (SubI zero (RShiftI src1 src2)));
11284 
11285   ins_cost(1.9 * INSN_COST);
11286   format %{ "negw  $dst, $src1, ASR $src2" %}
11287 
11288   ins_encode %{
11289     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11290             Assembler::ASR, $src2$$constant & 0x1f);
11291   %}
11292 
11293   ins_pipe(ialu_reg_shift);
11294 %}
11295 
11296 // This pattern is automatically generated from aarch64_ad.m4
11297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11298 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11299                               immI0 zero, iRegIorL2I src1, immI src2) %{
11300   match(Set dst (SubI zero (LShiftI src1 src2)));
11301 
11302   ins_cost(1.9 * INSN_COST);
11303   format %{ "negw  $dst, $src1, LSL $src2" %}
11304 
11305   ins_encode %{
11306     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11307             Assembler::LSL, $src2$$constant & 0x1f);
11308   %}
11309 
11310   ins_pipe(ialu_reg_shift);
11311 %}
11312 
11313 // This pattern is automatically generated from aarch64_ad.m4
11314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11315 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11316                               immL0 zero, iRegL src1, immI src2) %{
11317   match(Set dst (SubL zero (URShiftL src1 src2)));
11318 
11319   ins_cost(1.9 * INSN_COST);
11320   format %{ "neg  $dst, $src1, LSR $src2" %}
11321 
11322   ins_encode %{
11323     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11324             Assembler::LSR, $src2$$constant & 0x3f);
11325   %}
11326 
11327   ins_pipe(ialu_reg_shift);
11328 %}
11329 
11330 // This pattern is automatically generated from aarch64_ad.m4
11331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11332 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11333                               immL0 zero, iRegL src1, immI src2) %{
11334   match(Set dst (SubL zero (RShiftL src1 src2)));
11335 
11336   ins_cost(1.9 * INSN_COST);
11337   format %{ "neg  $dst, $src1, ASR $src2" %}
11338 
11339   ins_encode %{
11340     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11341             Assembler::ASR, $src2$$constant & 0x3f);
11342   %}
11343 
11344   ins_pipe(ialu_reg_shift);
11345 %}
11346 
11347 // This pattern is automatically generated from aarch64_ad.m4
11348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11349 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11350                               immL0 zero, iRegL src1, immI src2) %{
11351   match(Set dst (SubL zero (LShiftL src1 src2)));
11352 
11353   ins_cost(1.9 * INSN_COST);
11354   format %{ "neg  $dst, $src1, LSL $src2" %}
11355 
11356   ins_encode %{
11357     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11358             Assembler::LSL, $src2$$constant & 0x3f);
11359   %}
11360 
11361   ins_pipe(ialu_reg_shift);
11362 %}
11363 
11364 // This pattern is automatically generated from aarch64_ad.m4
11365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11366 instruct AndI_reg_not_reg(iRegINoSp dst,
11367                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11368   match(Set dst (AndI src1 (XorI src2 m1)));
11369   ins_cost(INSN_COST);
11370   format %{ "bicw  $dst, $src1, $src2" %}
11371 
11372   ins_encode %{
11373     __ bicw(as_Register($dst$$reg),
11374               as_Register($src1$$reg),
11375               as_Register($src2$$reg),
11376               Assembler::LSL, 0);
11377   %}
11378 
11379   ins_pipe(ialu_reg_reg);
11380 %}
11381 
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 instruct AndL_reg_not_reg(iRegLNoSp dst,
11385                          iRegL src1, iRegL src2, immL_M1 m1) %{
11386   match(Set dst (AndL src1 (XorL src2 m1)));
11387   ins_cost(INSN_COST);
11388   format %{ "bic  $dst, $src1, $src2" %}
11389 
11390   ins_encode %{
11391     __ bic(as_Register($dst$$reg),
11392               as_Register($src1$$reg),
11393               as_Register($src2$$reg),
11394               Assembler::LSL, 0);
11395   %}
11396 
11397   ins_pipe(ialu_reg_reg);
11398 %}
11399 
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 instruct OrI_reg_not_reg(iRegINoSp dst,
11403                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11404   match(Set dst (OrI src1 (XorI src2 m1)));
11405   ins_cost(INSN_COST);
11406   format %{ "ornw  $dst, $src1, $src2" %}
11407 
11408   ins_encode %{
11409     __ ornw(as_Register($dst$$reg),
11410               as_Register($src1$$reg),
11411               as_Register($src2$$reg),
11412               Assembler::LSL, 0);
11413   %}
11414 
11415   ins_pipe(ialu_reg_reg);
11416 %}
11417 
11418 // This pattern is automatically generated from aarch64_ad.m4
11419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11420 instruct OrL_reg_not_reg(iRegLNoSp dst,
11421                          iRegL src1, iRegL src2, immL_M1 m1) %{
11422   match(Set dst (OrL src1 (XorL src2 m1)));
11423   ins_cost(INSN_COST);
11424   format %{ "orn  $dst, $src1, $src2" %}
11425 
11426   ins_encode %{
11427     __ orn(as_Register($dst$$reg),
11428               as_Register($src1$$reg),
11429               as_Register($src2$$reg),
11430               Assembler::LSL, 0);
11431   %}
11432 
11433   ins_pipe(ialu_reg_reg);
11434 %}
11435 
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 instruct XorI_reg_not_reg(iRegINoSp dst,
11439                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11440   match(Set dst (XorI m1 (XorI src2 src1)));
11441   ins_cost(INSN_COST);
11442   format %{ "eonw  $dst, $src1, $src2" %}
11443 
11444   ins_encode %{
11445     __ eonw(as_Register($dst$$reg),
11446               as_Register($src1$$reg),
11447               as_Register($src2$$reg),
11448               Assembler::LSL, 0);
11449   %}
11450 
11451   ins_pipe(ialu_reg_reg);
11452 %}
11453 
11454 // This pattern is automatically generated from aarch64_ad.m4
11455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11456 instruct XorL_reg_not_reg(iRegLNoSp dst,
11457                          iRegL src1, iRegL src2, immL_M1 m1) %{
11458   match(Set dst (XorL m1 (XorL src2 src1)));
11459   ins_cost(INSN_COST);
11460   format %{ "eon  $dst, $src1, $src2" %}
11461 
11462   ins_encode %{
11463     __ eon(as_Register($dst$$reg),
11464               as_Register($src1$$reg),
11465               as_Register($src2$$reg),
11466               Assembler::LSL, 0);
11467   %}
11468 
11469   ins_pipe(ialu_reg_reg);
11470 %}
11471 
11472 // This pattern is automatically generated from aarch64_ad.m4
11473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11474 // val & (-1 ^ (val >>> shift)) ==> bicw
11475 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11476                          iRegIorL2I src1, iRegIorL2I src2,
11477                          immI src3, immI_M1 src4) %{
11478   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11479   ins_cost(1.9 * INSN_COST);
11480   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11481 
11482   ins_encode %{
11483     __ bicw(as_Register($dst$$reg),
11484               as_Register($src1$$reg),
11485               as_Register($src2$$reg),
11486               Assembler::LSR,
11487               $src3$$constant & 0x1f);
11488   %}
11489 
11490   ins_pipe(ialu_reg_reg_shift);
11491 %}
11492 
11493 // This pattern is automatically generated from aarch64_ad.m4
11494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11495 // val & (-1 ^ (val >>> shift)) ==> bic
11496 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11497                          iRegL src1, iRegL src2,
11498                          immI src3, immL_M1 src4) %{
11499   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11500   ins_cost(1.9 * INSN_COST);
11501   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11502 
11503   ins_encode %{
11504     __ bic(as_Register($dst$$reg),
11505               as_Register($src1$$reg),
11506               as_Register($src2$$reg),
11507               Assembler::LSR,
11508               $src3$$constant & 0x3f);
11509   %}
11510 
11511   ins_pipe(ialu_reg_reg_shift);
11512 %}
11513 
11514 // This pattern is automatically generated from aarch64_ad.m4
11515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11516 // val & (-1 ^ (val >> shift)) ==> bicw
11517 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11518                          iRegIorL2I src1, iRegIorL2I src2,
11519                          immI src3, immI_M1 src4) %{
11520   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11521   ins_cost(1.9 * INSN_COST);
11522   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11523 
11524   ins_encode %{
11525     __ bicw(as_Register($dst$$reg),
11526               as_Register($src1$$reg),
11527               as_Register($src2$$reg),
11528               Assembler::ASR,
11529               $src3$$constant & 0x1f);
11530   %}
11531 
11532   ins_pipe(ialu_reg_reg_shift);
11533 %}
11534 
11535 // This pattern is automatically generated from aarch64_ad.m4
11536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11537 // val & (-1 ^ (val >> shift)) ==> bic
11538 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11539                          iRegL src1, iRegL src2,
11540                          immI src3, immL_M1 src4) %{
11541   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11542   ins_cost(1.9 * INSN_COST);
11543   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11544 
11545   ins_encode %{
11546     __ bic(as_Register($dst$$reg),
11547               as_Register($src1$$reg),
11548               as_Register($src2$$reg),
11549               Assembler::ASR,
11550               $src3$$constant & 0x3f);
11551   %}
11552 
11553   ins_pipe(ialu_reg_reg_shift);
11554 %}
11555 
11556 // This pattern is automatically generated from aarch64_ad.m4
11557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11558 // val & (-1 ^ (val ror shift)) ==> bicw
11559 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11560                          iRegIorL2I src1, iRegIorL2I src2,
11561                          immI src3, immI_M1 src4) %{
11562   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11563   ins_cost(1.9 * INSN_COST);
11564   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11565 
11566   ins_encode %{
11567     __ bicw(as_Register($dst$$reg),
11568               as_Register($src1$$reg),
11569               as_Register($src2$$reg),
11570               Assembler::ROR,
11571               $src3$$constant & 0x1f);
11572   %}
11573 
11574   ins_pipe(ialu_reg_reg_shift);
11575 %}
11576 
11577 // This pattern is automatically generated from aarch64_ad.m4
11578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11579 // val & (-1 ^ (val ror shift)) ==> bic
11580 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11581                          iRegL src1, iRegL src2,
11582                          immI src3, immL_M1 src4) %{
11583   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11584   ins_cost(1.9 * INSN_COST);
11585   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11586 
11587   ins_encode %{
11588     __ bic(as_Register($dst$$reg),
11589               as_Register($src1$$reg),
11590               as_Register($src2$$reg),
11591               Assembler::ROR,
11592               $src3$$constant & 0x3f);
11593   %}
11594 
11595   ins_pipe(ialu_reg_reg_shift);
11596 %}
11597 
11598 // This pattern is automatically generated from aarch64_ad.m4
11599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11600 // val & (-1 ^ (val << shift)) ==> bicw
11601 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11602                          iRegIorL2I src1, iRegIorL2I src2,
11603                          immI src3, immI_M1 src4) %{
11604   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11605   ins_cost(1.9 * INSN_COST);
11606   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11607 
11608   ins_encode %{
11609     __ bicw(as_Register($dst$$reg),
11610               as_Register($src1$$reg),
11611               as_Register($src2$$reg),
11612               Assembler::LSL,
11613               $src3$$constant & 0x1f);
11614   %}
11615 
11616   ins_pipe(ialu_reg_reg_shift);
11617 %}
11618 
11619 // This pattern is automatically generated from aarch64_ad.m4
11620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11621 // val & (-1 ^ (val << shift)) ==> bic
11622 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11623                          iRegL src1, iRegL src2,
11624                          immI src3, immL_M1 src4) %{
11625   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11626   ins_cost(1.9 * INSN_COST);
11627   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11628 
11629   ins_encode %{
11630     __ bic(as_Register($dst$$reg),
11631               as_Register($src1$$reg),
11632               as_Register($src2$$reg),
11633               Assembler::LSL,
11634               $src3$$constant & 0x3f);
11635   %}
11636 
11637   ins_pipe(ialu_reg_reg_shift);
11638 %}
11639 
11640 // This pattern is automatically generated from aarch64_ad.m4
11641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11642 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11643 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11644                          iRegIorL2I src1, iRegIorL2I src2,
11645                          immI src3, immI_M1 src4) %{
11646   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11647   ins_cost(1.9 * INSN_COST);
11648   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11649 
11650   ins_encode %{
11651     __ eonw(as_Register($dst$$reg),
11652               as_Register($src1$$reg),
11653               as_Register($src2$$reg),
11654               Assembler::LSR,
11655               $src3$$constant & 0x1f);
11656   %}
11657 
11658   ins_pipe(ialu_reg_reg_shift);
11659 %}
11660 
11661 // This pattern is automatically generated from aarch64_ad.m4
11662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11663 // val ^ (-1 ^ (val >>> shift)) ==> eon
11664 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11665                          iRegL src1, iRegL src2,
11666                          immI src3, immL_M1 src4) %{
11667   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11668   ins_cost(1.9 * INSN_COST);
11669   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11670 
11671   ins_encode %{
11672     __ eon(as_Register($dst$$reg),
11673               as_Register($src1$$reg),
11674               as_Register($src2$$reg),
11675               Assembler::LSR,
11676               $src3$$constant & 0x3f);
11677   %}
11678 
11679   ins_pipe(ialu_reg_reg_shift);
11680 %}
11681 
11682 // This pattern is automatically generated from aarch64_ad.m4
11683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11684 // val ^ (-1 ^ (val >> shift)) ==> eonw
11685 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11686                          iRegIorL2I src1, iRegIorL2I src2,
11687                          immI src3, immI_M1 src4) %{
11688   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11689   ins_cost(1.9 * INSN_COST);
11690   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11691 
11692   ins_encode %{
11693     __ eonw(as_Register($dst$$reg),
11694               as_Register($src1$$reg),
11695               as_Register($src2$$reg),
11696               Assembler::ASR,
11697               $src3$$constant & 0x1f);
11698   %}
11699 
11700   ins_pipe(ialu_reg_reg_shift);
11701 %}
11702 
11703 // This pattern is automatically generated from aarch64_ad.m4
11704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11705 // val ^ (-1 ^ (val >> shift)) ==> eon
11706 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11707                          iRegL src1, iRegL src2,
11708                          immI src3, immL_M1 src4) %{
11709   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11710   ins_cost(1.9 * INSN_COST);
11711   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11712 
11713   ins_encode %{
11714     __ eon(as_Register($dst$$reg),
11715               as_Register($src1$$reg),
11716               as_Register($src2$$reg),
11717               Assembler::ASR,
11718               $src3$$constant & 0x3f);
11719   %}
11720 
11721   ins_pipe(ialu_reg_reg_shift);
11722 %}
11723 
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726 // val ^ (-1 ^ (val ror shift)) ==> eonw
11727 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11728                          iRegIorL2I src1, iRegIorL2I src2,
11729                          immI src3, immI_M1 src4) %{
11730   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11731   ins_cost(1.9 * INSN_COST);
11732   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11733 
11734   ins_encode %{
11735     __ eonw(as_Register($dst$$reg),
11736               as_Register($src1$$reg),
11737               as_Register($src2$$reg),
11738               Assembler::ROR,
11739               $src3$$constant & 0x1f);
11740   %}
11741 
11742   ins_pipe(ialu_reg_reg_shift);
11743 %}
11744 
11745 // This pattern is automatically generated from aarch64_ad.m4
11746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11747 // val ^ (-1 ^ (val ror shift)) ==> eon
11748 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11749                          iRegL src1, iRegL src2,
11750                          immI src3, immL_M1 src4) %{
11751   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11752   ins_cost(1.9 * INSN_COST);
11753   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11754 
11755   ins_encode %{
11756     __ eon(as_Register($dst$$reg),
11757               as_Register($src1$$reg),
11758               as_Register($src2$$reg),
11759               Assembler::ROR,
11760               $src3$$constant & 0x3f);
11761   %}
11762 
11763   ins_pipe(ialu_reg_reg_shift);
11764 %}
11765 
11766 // This pattern is automatically generated from aarch64_ad.m4
11767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11768 // val ^ (-1 ^ (val << shift)) ==> eonw
11769 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11770                          iRegIorL2I src1, iRegIorL2I src2,
11771                          immI src3, immI_M1 src4) %{
11772   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11773   ins_cost(1.9 * INSN_COST);
11774   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11775 
11776   ins_encode %{
11777     __ eonw(as_Register($dst$$reg),
11778               as_Register($src1$$reg),
11779               as_Register($src2$$reg),
11780               Assembler::LSL,
11781               $src3$$constant & 0x1f);
11782   %}
11783 
11784   ins_pipe(ialu_reg_reg_shift);
11785 %}
11786 
11787 // This pattern is automatically generated from aarch64_ad.m4
11788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11789 // val ^ (-1 ^ (val << shift)) ==> eon
11790 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11791                          iRegL src1, iRegL src2,
11792                          immI src3, immL_M1 src4) %{
11793   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11794   ins_cost(1.9 * INSN_COST);
11795   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11796 
11797   ins_encode %{
11798     __ eon(as_Register($dst$$reg),
11799               as_Register($src1$$reg),
11800               as_Register($src2$$reg),
11801               Assembler::LSL,
11802               $src3$$constant & 0x3f);
11803   %}
11804 
11805   ins_pipe(ialu_reg_reg_shift);
11806 %}
11807 
11808 // This pattern is automatically generated from aarch64_ad.m4
11809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11810 // val | (-1 ^ (val >>> shift)) ==> ornw
11811 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11812                          iRegIorL2I src1, iRegIorL2I src2,
11813                          immI src3, immI_M1 src4) %{
11814   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11815   ins_cost(1.9 * INSN_COST);
11816   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11817 
11818   ins_encode %{
11819     __ ornw(as_Register($dst$$reg),
11820               as_Register($src1$$reg),
11821               as_Register($src2$$reg),
11822               Assembler::LSR,
11823               $src3$$constant & 0x1f);
11824   %}
11825 
11826   ins_pipe(ialu_reg_reg_shift);
11827 %}
11828 
11829 // This pattern is automatically generated from aarch64_ad.m4
11830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11831 // val | (-1 ^ (val >>> shift)) ==> orn
11832 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11833                          iRegL src1, iRegL src2,
11834                          immI src3, immL_M1 src4) %{
11835   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11836   ins_cost(1.9 * INSN_COST);
11837   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11838 
11839   ins_encode %{
11840     __ orn(as_Register($dst$$reg),
11841               as_Register($src1$$reg),
11842               as_Register($src2$$reg),
11843               Assembler::LSR,
11844               $src3$$constant & 0x3f);
11845   %}
11846 
11847   ins_pipe(ialu_reg_reg_shift);
11848 %}
11849 
11850 // This pattern is automatically generated from aarch64_ad.m4
11851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11852 // val | (-1 ^ (val >> shift)) ==> ornw
11853 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11854                          iRegIorL2I src1, iRegIorL2I src2,
11855                          immI src3, immI_M1 src4) %{
11856   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11857   ins_cost(1.9 * INSN_COST);
11858   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11859 
11860   ins_encode %{
11861     __ ornw(as_Register($dst$$reg),
11862               as_Register($src1$$reg),
11863               as_Register($src2$$reg),
11864               Assembler::ASR,
11865               $src3$$constant & 0x1f);
11866   %}
11867 
11868   ins_pipe(ialu_reg_reg_shift);
11869 %}
11870 
11871 // This pattern is automatically generated from aarch64_ad.m4
11872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11873 // val | (-1 ^ (val >> shift)) ==> orn
11874 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11875                          iRegL src1, iRegL src2,
11876                          immI src3, immL_M1 src4) %{
11877   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11878   ins_cost(1.9 * INSN_COST);
11879   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11880 
11881   ins_encode %{
11882     __ orn(as_Register($dst$$reg),
11883               as_Register($src1$$reg),
11884               as_Register($src2$$reg),
11885               Assembler::ASR,
11886               $src3$$constant & 0x3f);
11887   %}
11888 
11889   ins_pipe(ialu_reg_reg_shift);
11890 %}
11891 
11892 // This pattern is automatically generated from aarch64_ad.m4
11893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11894 // val | (-1 ^ (val ror shift)) ==> ornw
11895 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11896                          iRegIorL2I src1, iRegIorL2I src2,
11897                          immI src3, immI_M1 src4) %{
11898   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11899   ins_cost(1.9 * INSN_COST);
11900   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11901 
11902   ins_encode %{
11903     __ ornw(as_Register($dst$$reg),
11904               as_Register($src1$$reg),
11905               as_Register($src2$$reg),
11906               Assembler::ROR,
11907               $src3$$constant & 0x1f);
11908   %}
11909 
11910   ins_pipe(ialu_reg_reg_shift);
11911 %}
11912 
11913 // This pattern is automatically generated from aarch64_ad.m4
11914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11915 // val | (-1 ^ (val ror shift)) ==> orn
11916 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11917                          iRegL src1, iRegL src2,
11918                          immI src3, immL_M1 src4) %{
11919   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11920   ins_cost(1.9 * INSN_COST);
11921   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11922 
11923   ins_encode %{
11924     __ orn(as_Register($dst$$reg),
11925               as_Register($src1$$reg),
11926               as_Register($src2$$reg),
11927               Assembler::ROR,
11928               $src3$$constant & 0x3f);
11929   %}
11930 
11931   ins_pipe(ialu_reg_reg_shift);
11932 %}
11933 
11934 // This pattern is automatically generated from aarch64_ad.m4
11935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11936 // val | (-1 ^ (val << shift)) ==> ornw
11937 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11938                          iRegIorL2I src1, iRegIorL2I src2,
11939                          immI src3, immI_M1 src4) %{
11940   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11941   ins_cost(1.9 * INSN_COST);
11942   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11943 
11944   ins_encode %{
11945     __ ornw(as_Register($dst$$reg),
11946               as_Register($src1$$reg),
11947               as_Register($src2$$reg),
11948               Assembler::LSL,
11949               $src3$$constant & 0x1f);
11950   %}
11951 
11952   ins_pipe(ialu_reg_reg_shift);
11953 %}
11954 
11955 // This pattern is automatically generated from aarch64_ad.m4
11956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11957 // val | (-1 ^ (val << shift)) ==> orn
11958 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11959                          iRegL src1, iRegL src2,
11960                          immI src3, immL_M1 src4) %{
11961   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11962   ins_cost(1.9 * INSN_COST);
11963   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11964 
11965   ins_encode %{
11966     __ orn(as_Register($dst$$reg),
11967               as_Register($src1$$reg),
11968               as_Register($src2$$reg),
11969               Assembler::LSL,
11970               $src3$$constant & 0x3f);
11971   %}
11972 
11973   ins_pipe(ialu_reg_reg_shift);
11974 %}
11975 
11976 // This pattern is automatically generated from aarch64_ad.m4
11977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11978 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11979                          iRegIorL2I src1, iRegIorL2I src2,
11980                          immI src3) %{
11981   match(Set dst (AndI src1 (URShiftI src2 src3)));
11982 
11983   ins_cost(1.9 * INSN_COST);
11984   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11985 
11986   ins_encode %{
11987     __ andw(as_Register($dst$$reg),
11988               as_Register($src1$$reg),
11989               as_Register($src2$$reg),
11990               Assembler::LSR,
11991               $src3$$constant & 0x1f);
11992   %}
11993 
11994   ins_pipe(ialu_reg_reg_shift);
11995 %}
11996 
11997 // This pattern is automatically generated from aarch64_ad.m4
11998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11999 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12000                          iRegL src1, iRegL src2,
12001                          immI src3) %{
12002   match(Set dst (AndL src1 (URShiftL src2 src3)));
12003 
12004   ins_cost(1.9 * INSN_COST);
12005   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12006 
12007   ins_encode %{
12008     __ andr(as_Register($dst$$reg),
12009               as_Register($src1$$reg),
12010               as_Register($src2$$reg),
12011               Assembler::LSR,
12012               $src3$$constant & 0x3f);
12013   %}
12014 
12015   ins_pipe(ialu_reg_reg_shift);
12016 %}
12017 
12018 // This pattern is automatically generated from aarch64_ad.m4
12019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12020 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12021                          iRegIorL2I src1, iRegIorL2I src2,
12022                          immI src3) %{
12023   match(Set dst (AndI src1 (RShiftI src2 src3)));
12024 
12025   ins_cost(1.9 * INSN_COST);
12026   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12027 
12028   ins_encode %{
12029     __ andw(as_Register($dst$$reg),
12030               as_Register($src1$$reg),
12031               as_Register($src2$$reg),
12032               Assembler::ASR,
12033               $src3$$constant & 0x1f);
12034   %}
12035 
12036   ins_pipe(ialu_reg_reg_shift);
12037 %}
12038 
12039 // This pattern is automatically generated from aarch64_ad.m4
12040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12041 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12042                          iRegL src1, iRegL src2,
12043                          immI src3) %{
12044   match(Set dst (AndL src1 (RShiftL src2 src3)));
12045 
12046   ins_cost(1.9 * INSN_COST);
12047   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12048 
12049   ins_encode %{
12050     __ andr(as_Register($dst$$reg),
12051               as_Register($src1$$reg),
12052               as_Register($src2$$reg),
12053               Assembler::ASR,
12054               $src3$$constant & 0x3f);
12055   %}
12056 
12057   ins_pipe(ialu_reg_reg_shift);
12058 %}
12059 
12060 // This pattern is automatically generated from aarch64_ad.m4
12061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12062 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12063                          iRegIorL2I src1, iRegIorL2I src2,
12064                          immI src3) %{
12065   match(Set dst (AndI src1 (LShiftI src2 src3)));
12066 
12067   ins_cost(1.9 * INSN_COST);
12068   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12069 
12070   ins_encode %{
12071     __ andw(as_Register($dst$$reg),
12072               as_Register($src1$$reg),
12073               as_Register($src2$$reg),
12074               Assembler::LSL,
12075               $src3$$constant & 0x1f);
12076   %}
12077 
12078   ins_pipe(ialu_reg_reg_shift);
12079 %}
12080 
12081 // This pattern is automatically generated from aarch64_ad.m4
12082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12083 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12084                          iRegL src1, iRegL src2,
12085                          immI src3) %{
12086   match(Set dst (AndL src1 (LShiftL src2 src3)));
12087 
12088   ins_cost(1.9 * INSN_COST);
12089   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12090 
12091   ins_encode %{
12092     __ andr(as_Register($dst$$reg),
12093               as_Register($src1$$reg),
12094               as_Register($src2$$reg),
12095               Assembler::LSL,
12096               $src3$$constant & 0x3f);
12097   %}
12098 
12099   ins_pipe(ialu_reg_reg_shift);
12100 %}
12101 
12102 // This pattern is automatically generated from aarch64_ad.m4
12103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12104 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12105                          iRegIorL2I src1, iRegIorL2I src2,
12106                          immI src3) %{
12107   match(Set dst (AndI src1 (RotateRight src2 src3)));
12108 
12109   ins_cost(1.9 * INSN_COST);
12110   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12111 
12112   ins_encode %{
12113     __ andw(as_Register($dst$$reg),
12114               as_Register($src1$$reg),
12115               as_Register($src2$$reg),
12116               Assembler::ROR,
12117               $src3$$constant & 0x1f);
12118   %}
12119 
12120   ins_pipe(ialu_reg_reg_shift);
12121 %}
12122 
12123 // This pattern is automatically generated from aarch64_ad.m4
12124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12125 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12126                          iRegL src1, iRegL src2,
12127                          immI src3) %{
12128   match(Set dst (AndL src1 (RotateRight src2 src3)));
12129 
12130   ins_cost(1.9 * INSN_COST);
12131   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12132 
12133   ins_encode %{
12134     __ andr(as_Register($dst$$reg),
12135               as_Register($src1$$reg),
12136               as_Register($src2$$reg),
12137               Assembler::ROR,
12138               $src3$$constant & 0x3f);
12139   %}
12140 
12141   ins_pipe(ialu_reg_reg_shift);
12142 %}
12143 
12144 // This pattern is automatically generated from aarch64_ad.m4
12145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12146 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12147                          iRegIorL2I src1, iRegIorL2I src2,
12148                          immI src3) %{
12149   match(Set dst (XorI src1 (URShiftI src2 src3)));
12150 
12151   ins_cost(1.9 * INSN_COST);
12152   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12153 
12154   ins_encode %{
12155     __ eorw(as_Register($dst$$reg),
12156               as_Register($src1$$reg),
12157               as_Register($src2$$reg),
12158               Assembler::LSR,
12159               $src3$$constant & 0x1f);
12160   %}
12161 
12162   ins_pipe(ialu_reg_reg_shift);
12163 %}
12164 
12165 // This pattern is automatically generated from aarch64_ad.m4
12166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12167 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12168                          iRegL src1, iRegL src2,
12169                          immI src3) %{
12170   match(Set dst (XorL src1 (URShiftL src2 src3)));
12171 
12172   ins_cost(1.9 * INSN_COST);
12173   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12174 
12175   ins_encode %{
12176     __ eor(as_Register($dst$$reg),
12177               as_Register($src1$$reg),
12178               as_Register($src2$$reg),
12179               Assembler::LSR,
12180               $src3$$constant & 0x3f);
12181   %}
12182 
12183   ins_pipe(ialu_reg_reg_shift);
12184 %}
12185 
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12189                          iRegIorL2I src1, iRegIorL2I src2,
12190                          immI src3) %{
12191   match(Set dst (XorI src1 (RShiftI src2 src3)));
12192 
12193   ins_cost(1.9 * INSN_COST);
12194   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12195 
12196   ins_encode %{
12197     __ eorw(as_Register($dst$$reg),
12198               as_Register($src1$$reg),
12199               as_Register($src2$$reg),
12200               Assembler::ASR,
12201               $src3$$constant & 0x1f);
12202   %}
12203 
12204   ins_pipe(ialu_reg_reg_shift);
12205 %}
12206 
12207 // This pattern is automatically generated from aarch64_ad.m4
12208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12209 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12210                          iRegL src1, iRegL src2,
12211                          immI src3) %{
12212   match(Set dst (XorL src1 (RShiftL src2 src3)));
12213 
12214   ins_cost(1.9 * INSN_COST);
12215   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12216 
12217   ins_encode %{
12218     __ eor(as_Register($dst$$reg),
12219               as_Register($src1$$reg),
12220               as_Register($src2$$reg),
12221               Assembler::ASR,
12222               $src3$$constant & 0x3f);
12223   %}
12224 
12225   ins_pipe(ialu_reg_reg_shift);
12226 %}
12227 
12228 // This pattern is automatically generated from aarch64_ad.m4
12229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12230 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12231                          iRegIorL2I src1, iRegIorL2I src2,
12232                          immI src3) %{
12233   match(Set dst (XorI src1 (LShiftI src2 src3)));
12234 
12235   ins_cost(1.9 * INSN_COST);
12236   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12237 
12238   ins_encode %{
12239     __ eorw(as_Register($dst$$reg),
12240               as_Register($src1$$reg),
12241               as_Register($src2$$reg),
12242               Assembler::LSL,
12243               $src3$$constant & 0x1f);
12244   %}
12245 
12246   ins_pipe(ialu_reg_reg_shift);
12247 %}
12248 
12249 // This pattern is automatically generated from aarch64_ad.m4
12250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12251 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12252                          iRegL src1, iRegL src2,
12253                          immI src3) %{
12254   match(Set dst (XorL src1 (LShiftL src2 src3)));
12255 
12256   ins_cost(1.9 * INSN_COST);
12257   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12258 
12259   ins_encode %{
12260     __ eor(as_Register($dst$$reg),
12261               as_Register($src1$$reg),
12262               as_Register($src2$$reg),
12263               Assembler::LSL,
12264               $src3$$constant & 0x3f);
12265   %}
12266 
12267   ins_pipe(ialu_reg_reg_shift);
12268 %}
12269 
12270 // This pattern is automatically generated from aarch64_ad.m4
12271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12272 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12273                          iRegIorL2I src1, iRegIorL2I src2,
12274                          immI src3) %{
12275   match(Set dst (XorI src1 (RotateRight src2 src3)));
12276 
12277   ins_cost(1.9 * INSN_COST);
12278   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12279 
12280   ins_encode %{
12281     __ eorw(as_Register($dst$$reg),
12282               as_Register($src1$$reg),
12283               as_Register($src2$$reg),
12284               Assembler::ROR,
12285               $src3$$constant & 0x1f);
12286   %}
12287 
12288   ins_pipe(ialu_reg_reg_shift);
12289 %}
12290 
12291 // This pattern is automatically generated from aarch64_ad.m4
12292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12293 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12294                          iRegL src1, iRegL src2,
12295                          immI src3) %{
12296   match(Set dst (XorL src1 (RotateRight src2 src3)));
12297 
12298   ins_cost(1.9 * INSN_COST);
12299   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12300 
12301   ins_encode %{
12302     __ eor(as_Register($dst$$reg),
12303               as_Register($src1$$reg),
12304               as_Register($src2$$reg),
12305               Assembler::ROR,
12306               $src3$$constant & 0x3f);
12307   %}
12308 
12309   ins_pipe(ialu_reg_reg_shift);
12310 %}
12311 
12312 // This pattern is automatically generated from aarch64_ad.m4
12313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12314 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12315                          iRegIorL2I src1, iRegIorL2I src2,
12316                          immI src3) %{
12317   match(Set dst (OrI src1 (URShiftI src2 src3)));
12318 
12319   ins_cost(1.9 * INSN_COST);
12320   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12321 
12322   ins_encode %{
12323     __ orrw(as_Register($dst$$reg),
12324               as_Register($src1$$reg),
12325               as_Register($src2$$reg),
12326               Assembler::LSR,
12327               $src3$$constant & 0x1f);
12328   %}
12329 
12330   ins_pipe(ialu_reg_reg_shift);
12331 %}
12332 
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12336                          iRegL src1, iRegL src2,
12337                          immI src3) %{
12338   match(Set dst (OrL src1 (URShiftL src2 src3)));
12339 
12340   ins_cost(1.9 * INSN_COST);
12341   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12342 
12343   ins_encode %{
12344     __ orr(as_Register($dst$$reg),
12345               as_Register($src1$$reg),
12346               as_Register($src2$$reg),
12347               Assembler::LSR,
12348               $src3$$constant & 0x3f);
12349   %}
12350 
12351   ins_pipe(ialu_reg_reg_shift);
12352 %}
12353 
12354 // This pattern is automatically generated from aarch64_ad.m4
12355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12356 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12357                          iRegIorL2I src1, iRegIorL2I src2,
12358                          immI src3) %{
12359   match(Set dst (OrI src1 (RShiftI src2 src3)));
12360 
12361   ins_cost(1.9 * INSN_COST);
12362   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12363 
12364   ins_encode %{
12365     __ orrw(as_Register($dst$$reg),
12366               as_Register($src1$$reg),
12367               as_Register($src2$$reg),
12368               Assembler::ASR,
12369               $src3$$constant & 0x1f);
12370   %}
12371 
12372   ins_pipe(ialu_reg_reg_shift);
12373 %}
12374 
12375 // This pattern is automatically generated from aarch64_ad.m4
12376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12377 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12378                          iRegL src1, iRegL src2,
12379                          immI src3) %{
12380   match(Set dst (OrL src1 (RShiftL src2 src3)));
12381 
12382   ins_cost(1.9 * INSN_COST);
12383   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12384 
12385   ins_encode %{
12386     __ orr(as_Register($dst$$reg),
12387               as_Register($src1$$reg),
12388               as_Register($src2$$reg),
12389               Assembler::ASR,
12390               $src3$$constant & 0x3f);
12391   %}
12392 
12393   ins_pipe(ialu_reg_reg_shift);
12394 %}
12395 
12396 // This pattern is automatically generated from aarch64_ad.m4
12397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12398 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12399                          iRegIorL2I src1, iRegIorL2I src2,
12400                          immI src3) %{
12401   match(Set dst (OrI src1 (LShiftI src2 src3)));
12402 
12403   ins_cost(1.9 * INSN_COST);
12404   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12405 
12406   ins_encode %{
12407     __ orrw(as_Register($dst$$reg),
12408               as_Register($src1$$reg),
12409               as_Register($src2$$reg),
12410               Assembler::LSL,
12411               $src3$$constant & 0x1f);
12412   %}
12413 
12414   ins_pipe(ialu_reg_reg_shift);
12415 %}
12416 
12417 // This pattern is automatically generated from aarch64_ad.m4
12418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12419 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12420                          iRegL src1, iRegL src2,
12421                          immI src3) %{
12422   match(Set dst (OrL src1 (LShiftL src2 src3)));
12423 
12424   ins_cost(1.9 * INSN_COST);
12425   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12426 
12427   ins_encode %{
12428     __ orr(as_Register($dst$$reg),
12429               as_Register($src1$$reg),
12430               as_Register($src2$$reg),
12431               Assembler::LSL,
12432               $src3$$constant & 0x3f);
12433   %}
12434 
12435   ins_pipe(ialu_reg_reg_shift);
12436 %}
12437 
12438 // This pattern is automatically generated from aarch64_ad.m4
12439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12440 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12441                          iRegIorL2I src1, iRegIorL2I src2,
12442                          immI src3) %{
12443   match(Set dst (OrI src1 (RotateRight src2 src3)));
12444 
12445   ins_cost(1.9 * INSN_COST);
12446   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12447 
12448   ins_encode %{
12449     __ orrw(as_Register($dst$$reg),
12450               as_Register($src1$$reg),
12451               as_Register($src2$$reg),
12452               Assembler::ROR,
12453               $src3$$constant & 0x1f);
12454   %}
12455 
12456   ins_pipe(ialu_reg_reg_shift);
12457 %}
12458 
12459 // This pattern is automatically generated from aarch64_ad.m4
12460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12461 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12462                          iRegL src1, iRegL src2,
12463                          immI src3) %{
12464   match(Set dst (OrL src1 (RotateRight src2 src3)));
12465 
12466   ins_cost(1.9 * INSN_COST);
12467   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12468 
12469   ins_encode %{
12470     __ orr(as_Register($dst$$reg),
12471               as_Register($src1$$reg),
12472               as_Register($src2$$reg),
12473               Assembler::ROR,
12474               $src3$$constant & 0x3f);
12475   %}
12476 
12477   ins_pipe(ialu_reg_reg_shift);
12478 %}
12479 
12480 // This pattern is automatically generated from aarch64_ad.m4
12481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12482 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12483                          iRegIorL2I src1, iRegIorL2I src2,
12484                          immI src3) %{
12485   match(Set dst (AddI src1 (URShiftI src2 src3)));
12486 
12487   ins_cost(1.9 * INSN_COST);
12488   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12489 
12490   ins_encode %{
12491     __ addw(as_Register($dst$$reg),
12492               as_Register($src1$$reg),
12493               as_Register($src2$$reg),
12494               Assembler::LSR,
12495               $src3$$constant & 0x1f);
12496   %}
12497 
12498   ins_pipe(ialu_reg_reg_shift);
12499 %}
12500 
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12504                          iRegL src1, iRegL src2,
12505                          immI src3) %{
12506   match(Set dst (AddL src1 (URShiftL src2 src3)));
12507 
12508   ins_cost(1.9 * INSN_COST);
12509   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12510 
12511   ins_encode %{
12512     __ add(as_Register($dst$$reg),
12513               as_Register($src1$$reg),
12514               as_Register($src2$$reg),
12515               Assembler::LSR,
12516               $src3$$constant & 0x3f);
12517   %}
12518 
12519   ins_pipe(ialu_reg_reg_shift);
12520 %}
12521 
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12525                          iRegIorL2I src1, iRegIorL2I src2,
12526                          immI src3) %{
12527   match(Set dst (AddI src1 (RShiftI src2 src3)));
12528 
12529   ins_cost(1.9 * INSN_COST);
12530   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12531 
12532   ins_encode %{
12533     __ addw(as_Register($dst$$reg),
12534               as_Register($src1$$reg),
12535               as_Register($src2$$reg),
12536               Assembler::ASR,
12537               $src3$$constant & 0x1f);
12538   %}
12539 
12540   ins_pipe(ialu_reg_reg_shift);
12541 %}
12542 
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12546                          iRegL src1, iRegL src2,
12547                          immI src3) %{
12548   match(Set dst (AddL src1 (RShiftL src2 src3)));
12549 
12550   ins_cost(1.9 * INSN_COST);
12551   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12552 
12553   ins_encode %{
12554     __ add(as_Register($dst$$reg),
12555               as_Register($src1$$reg),
12556               as_Register($src2$$reg),
12557               Assembler::ASR,
12558               $src3$$constant & 0x3f);
12559   %}
12560 
12561   ins_pipe(ialu_reg_reg_shift);
12562 %}
12563 
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12567                          iRegIorL2I src1, iRegIorL2I src2,
12568                          immI src3) %{
12569   match(Set dst (AddI src1 (LShiftI src2 src3)));
12570 
12571   ins_cost(1.9 * INSN_COST);
12572   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12573 
12574   ins_encode %{
12575     __ addw(as_Register($dst$$reg),
12576               as_Register($src1$$reg),
12577               as_Register($src2$$reg),
12578               Assembler::LSL,
12579               $src3$$constant & 0x1f);
12580   %}
12581 
12582   ins_pipe(ialu_reg_reg_shift);
12583 %}
12584 
12585 // This pattern is automatically generated from aarch64_ad.m4
12586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12587 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12588                          iRegL src1, iRegL src2,
12589                          immI src3) %{
12590   match(Set dst (AddL src1 (LShiftL src2 src3)));
12591 
12592   ins_cost(1.9 * INSN_COST);
12593   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12594 
12595   ins_encode %{
12596     __ add(as_Register($dst$$reg),
12597               as_Register($src1$$reg),
12598               as_Register($src2$$reg),
12599               Assembler::LSL,
12600               $src3$$constant & 0x3f);
12601   %}
12602 
12603   ins_pipe(ialu_reg_reg_shift);
12604 %}
12605 
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12609                          iRegIorL2I src1, iRegIorL2I src2,
12610                          immI src3) %{
12611   match(Set dst (SubI src1 (URShiftI src2 src3)));
12612 
12613   ins_cost(1.9 * INSN_COST);
12614   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12615 
12616   ins_encode %{
12617     __ subw(as_Register($dst$$reg),
12618               as_Register($src1$$reg),
12619               as_Register($src2$$reg),
12620               Assembler::LSR,
12621               $src3$$constant & 0x1f);
12622   %}
12623 
12624   ins_pipe(ialu_reg_reg_shift);
12625 %}
12626 
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12630                          iRegL src1, iRegL src2,
12631                          immI src3) %{
12632   match(Set dst (SubL src1 (URShiftL src2 src3)));
12633 
12634   ins_cost(1.9 * INSN_COST);
12635   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12636 
12637   ins_encode %{
12638     __ sub(as_Register($dst$$reg),
12639               as_Register($src1$$reg),
12640               as_Register($src2$$reg),
12641               Assembler::LSR,
12642               $src3$$constant & 0x3f);
12643   %}
12644 
12645   ins_pipe(ialu_reg_reg_shift);
12646 %}
12647 
12648 // This pattern is automatically generated from aarch64_ad.m4
12649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12650 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12651                          iRegIorL2I src1, iRegIorL2I src2,
12652                          immI src3) %{
12653   match(Set dst (SubI src1 (RShiftI src2 src3)));
12654 
12655   ins_cost(1.9 * INSN_COST);
12656   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12657 
12658   ins_encode %{
12659     __ subw(as_Register($dst$$reg),
12660               as_Register($src1$$reg),
12661               as_Register($src2$$reg),
12662               Assembler::ASR,
12663               $src3$$constant & 0x1f);
12664   %}
12665 
12666   ins_pipe(ialu_reg_reg_shift);
12667 %}
12668 
12669 // This pattern is automatically generated from aarch64_ad.m4
12670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12671 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12672                          iRegL src1, iRegL src2,
12673                          immI src3) %{
12674   match(Set dst (SubL src1 (RShiftL src2 src3)));
12675 
12676   ins_cost(1.9 * INSN_COST);
12677   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12678 
12679   ins_encode %{
12680     __ sub(as_Register($dst$$reg),
12681               as_Register($src1$$reg),
12682               as_Register($src2$$reg),
12683               Assembler::ASR,
12684               $src3$$constant & 0x3f);
12685   %}
12686 
12687   ins_pipe(ialu_reg_reg_shift);
12688 %}
12689 
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12693                          iRegIorL2I src1, iRegIorL2I src2,
12694                          immI src3) %{
12695   match(Set dst (SubI src1 (LShiftI src2 src3)));
12696 
12697   ins_cost(1.9 * INSN_COST);
12698   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12699 
12700   ins_encode %{
12701     __ subw(as_Register($dst$$reg),
12702               as_Register($src1$$reg),
12703               as_Register($src2$$reg),
12704               Assembler::LSL,
12705               $src3$$constant & 0x1f);
12706   %}
12707 
12708   ins_pipe(ialu_reg_reg_shift);
12709 %}
12710 
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12714                          iRegL src1, iRegL src2,
12715                          immI src3) %{
12716   match(Set dst (SubL src1 (LShiftL src2 src3)));
12717 
12718   ins_cost(1.9 * INSN_COST);
12719   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12720 
12721   ins_encode %{
12722     __ sub(as_Register($dst$$reg),
12723               as_Register($src1$$reg),
12724               as_Register($src2$$reg),
12725               Assembler::LSL,
12726               $src3$$constant & 0x3f);
12727   %}
12728 
12729   ins_pipe(ialu_reg_reg_shift);
12730 %}
12731 
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 
12735 // Shift Left followed by Shift Right.
12736 // This idiom is used by the compiler for the i2b bytecode etc.
12737 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12738 %{
12739   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12740   ins_cost(INSN_COST * 2);
12741   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12742   ins_encode %{
12743     int lshift = $lshift_count$$constant & 63;
12744     int rshift = $rshift_count$$constant & 63;
12745     int s = 63 - lshift;
12746     int r = (rshift - lshift) & 63;
12747     __ sbfm(as_Register($dst$$reg),
12748             as_Register($src$$reg),
12749             r, s);
12750   %}
12751 
12752   ins_pipe(ialu_reg_shift);
12753 %}
12754 
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 
12758 // Shift Left followed by Shift Right.
12759 // This idiom is used by the compiler for the i2b bytecode etc.
12760 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12761 %{
12762   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12763   ins_cost(INSN_COST * 2);
12764   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12765   ins_encode %{
12766     int lshift = $lshift_count$$constant & 31;
12767     int rshift = $rshift_count$$constant & 31;
12768     int s = 31 - lshift;
12769     int r = (rshift - lshift) & 31;
12770     __ sbfmw(as_Register($dst$$reg),
12771             as_Register($src$$reg),
12772             r, s);
12773   %}
12774 
12775   ins_pipe(ialu_reg_shift);
12776 %}
12777 
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 
12781 // Shift Left followed by Shift Right.
12782 // This idiom is used by the compiler for the i2b bytecode etc.
12783 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12784 %{
12785   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12786   ins_cost(INSN_COST * 2);
12787   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12788   ins_encode %{
12789     int lshift = $lshift_count$$constant & 63;
12790     int rshift = $rshift_count$$constant & 63;
12791     int s = 63 - lshift;
12792     int r = (rshift - lshift) & 63;
12793     __ ubfm(as_Register($dst$$reg),
12794             as_Register($src$$reg),
12795             r, s);
12796   %}
12797 
12798   ins_pipe(ialu_reg_shift);
12799 %}
12800 
12801 // This pattern is automatically generated from aarch64_ad.m4
12802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12803 
12804 // Shift Left followed by Shift Right.
12805 // This idiom is used by the compiler for the i2b bytecode etc.
12806 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12807 %{
12808   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12809   ins_cost(INSN_COST * 2);
12810   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12811   ins_encode %{
12812     int lshift = $lshift_count$$constant & 31;
12813     int rshift = $rshift_count$$constant & 31;
12814     int s = 31 - lshift;
12815     int r = (rshift - lshift) & 31;
12816     __ ubfmw(as_Register($dst$$reg),
12817             as_Register($src$$reg),
12818             r, s);
12819   %}
12820 
12821   ins_pipe(ialu_reg_shift);
12822 %}
12823 
12824 // Bitfield extract with shift & mask
12825 
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12829 %{
12830   match(Set dst (AndI (URShiftI src rshift) mask));
12831   // Make sure we are not going to exceed what ubfxw can do.
12832   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12833 
12834   ins_cost(INSN_COST);
12835   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12836   ins_encode %{
12837     int rshift = $rshift$$constant & 31;
12838     intptr_t mask = $mask$$constant;
12839     int width = exact_log2(mask+1);
12840     __ ubfxw(as_Register($dst$$reg),
12841             as_Register($src$$reg), rshift, width);
12842   %}
12843   ins_pipe(ialu_reg_shift);
12844 %}
12845 
12846 // This pattern is automatically generated from aarch64_ad.m4
12847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12848 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12849 %{
12850   match(Set dst (AndL (URShiftL src rshift) mask));
12851   // Make sure we are not going to exceed what ubfx can do.
12852   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12853 
12854   ins_cost(INSN_COST);
12855   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12856   ins_encode %{
12857     int rshift = $rshift$$constant & 63;
12858     intptr_t mask = $mask$$constant;
12859     int width = exact_log2_long(mask+1);
12860     __ ubfx(as_Register($dst$$reg),
12861             as_Register($src$$reg), rshift, width);
12862   %}
12863   ins_pipe(ialu_reg_shift);
12864 %}
12865 
12866 
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 
12870 // We can use ubfx when extending an And with a mask when we know mask
12871 // is positive.  We know that because immI_bitmask guarantees it.
12872 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12873 %{
12874   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12875   // Make sure we are not going to exceed what ubfxw can do.
12876   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12877 
12878   ins_cost(INSN_COST * 2);
12879   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12880   ins_encode %{
12881     int rshift = $rshift$$constant & 31;
12882     intptr_t mask = $mask$$constant;
12883     int width = exact_log2(mask+1);
12884     __ ubfx(as_Register($dst$$reg),
12885             as_Register($src$$reg), rshift, width);
12886   %}
12887   ins_pipe(ialu_reg_shift);
12888 %}
12889 
12890 
12891 // This pattern is automatically generated from aarch64_ad.m4
12892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12893 
12894 // We can use ubfiz when masking by a positive number and then left shifting the result.
12895 // We know that the mask is positive because immI_bitmask guarantees it.
12896 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12897 %{
12898   match(Set dst (LShiftI (AndI src mask) lshift));
12899   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12900 
12901   ins_cost(INSN_COST);
12902   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12903   ins_encode %{
12904     int lshift = $lshift$$constant & 31;
12905     intptr_t mask = $mask$$constant;
12906     int width = exact_log2(mask+1);
12907     __ ubfizw(as_Register($dst$$reg),
12908           as_Register($src$$reg), lshift, width);
12909   %}
12910   ins_pipe(ialu_reg_shift);
12911 %}
12912 
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 
12916 // We can use ubfiz when masking by a positive number and then left shifting the result.
12917 // We know that the mask is positive because immL_bitmask guarantees it.
12918 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12919 %{
12920   match(Set dst (LShiftL (AndL src mask) lshift));
12921   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12922 
12923   ins_cost(INSN_COST);
12924   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12925   ins_encode %{
12926     int lshift = $lshift$$constant & 63;
12927     intptr_t mask = $mask$$constant;
12928     int width = exact_log2_long(mask+1);
12929     __ ubfiz(as_Register($dst$$reg),
12930           as_Register($src$$reg), lshift, width);
12931   %}
12932   ins_pipe(ialu_reg_shift);
12933 %}
12934 
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937 
12938 // We can use ubfiz when masking by a positive number and then left shifting the result.
12939 // We know that the mask is positive because immI_bitmask guarantees it.
12940 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12941 %{
12942   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12943   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12944 
12945   ins_cost(INSN_COST);
12946   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12947   ins_encode %{
12948     int lshift = $lshift$$constant & 31;
12949     intptr_t mask = $mask$$constant;
12950     int width = exact_log2(mask+1);
12951     __ ubfizw(as_Register($dst$$reg),
12952           as_Register($src$$reg), lshift, width);
12953   %}
12954   ins_pipe(ialu_reg_shift);
12955 %}
12956 
12957 // This pattern is automatically generated from aarch64_ad.m4
12958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12959 
12960 // We can use ubfiz when masking by a positive number and then left shifting the result.
12961 // We know that the mask is positive because immL_bitmask guarantees it.
12962 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12963 %{
12964   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12965   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12966 
12967   ins_cost(INSN_COST);
12968   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12969   ins_encode %{
12970     int lshift = $lshift$$constant & 63;
12971     intptr_t mask = $mask$$constant;
12972     int width = exact_log2_long(mask+1);
12973     __ ubfiz(as_Register($dst$$reg),
12974           as_Register($src$$reg), lshift, width);
12975   %}
12976   ins_pipe(ialu_reg_shift);
12977 %}
12978 
12979 
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 
12983 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12984 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12985 %{
12986   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12987   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12988 
12989   ins_cost(INSN_COST);
12990   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12991   ins_encode %{
12992     int lshift = $lshift$$constant & 63;
12993     intptr_t mask = $mask$$constant;
12994     int width = exact_log2(mask+1);
12995     __ ubfiz(as_Register($dst$$reg),
12996              as_Register($src$$reg), lshift, width);
12997   %}
12998   ins_pipe(ialu_reg_shift);
12999 %}
13000 
13001 // This pattern is automatically generated from aarch64_ad.m4
13002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13003 
13004 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13005 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13006 %{
13007   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13008   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13009 
13010   ins_cost(INSN_COST);
13011   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13012   ins_encode %{
13013     int lshift = $lshift$$constant & 31;
13014     intptr_t mask = $mask$$constant;
13015     int width = exact_log2(mask+1);
13016     __ ubfiz(as_Register($dst$$reg),
13017              as_Register($src$$reg), lshift, width);
13018   %}
13019   ins_pipe(ialu_reg_shift);
13020 %}
13021 
13022 // This pattern is automatically generated from aarch64_ad.m4
13023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13024 
13025 // Can skip int2long conversions after AND with small bitmask
13026 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13027 %{
13028   match(Set dst (ConvI2L (AndI src msk)));
13029   ins_cost(INSN_COST);
13030   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13031   ins_encode %{
13032     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13033   %}
13034   ins_pipe(ialu_reg_shift);
13035 %}
13036 
13037 
13038 // Rotations
13039 
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13043 %{
13044   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13045   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13046 
13047   ins_cost(INSN_COST);
13048   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13049 
13050   ins_encode %{
13051     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13052             $rshift$$constant & 63);
13053   %}
13054   ins_pipe(ialu_reg_reg_extr);
13055 %}
13056 
13057 
13058 // This pattern is automatically generated from aarch64_ad.m4
13059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13060 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13061 %{
13062   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13063   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13064 
13065   ins_cost(INSN_COST);
13066   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13067 
13068   ins_encode %{
13069     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13070             $rshift$$constant & 31);
13071   %}
13072   ins_pipe(ialu_reg_reg_extr);
13073 %}
13074 
13075 
13076 // This pattern is automatically generated from aarch64_ad.m4
13077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13078 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13079 %{
13080   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13081   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13082 
13083   ins_cost(INSN_COST);
13084   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13085 
13086   ins_encode %{
13087     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13088             $rshift$$constant & 63);
13089   %}
13090   ins_pipe(ialu_reg_reg_extr);
13091 %}
13092 
13093 
13094 // This pattern is automatically generated from aarch64_ad.m4
13095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13096 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13097 %{
13098   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13099   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13100 
13101   ins_cost(INSN_COST);
13102   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13103 
13104   ins_encode %{
13105     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13106             $rshift$$constant & 31);
13107   %}
13108   ins_pipe(ialu_reg_reg_extr);
13109 %}
13110 
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13114 %{
13115   match(Set dst (RotateRight src shift));
13116 
13117   ins_cost(INSN_COST);
13118   format %{ "ror    $dst, $src, $shift" %}
13119 
13120   ins_encode %{
13121      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13122                $shift$$constant & 0x1f);
13123   %}
13124   ins_pipe(ialu_reg_reg_vshift);
13125 %}
13126 
13127 // This pattern is automatically generated from aarch64_ad.m4
13128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13129 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13130 %{
13131   match(Set dst (RotateRight src shift));
13132 
13133   ins_cost(INSN_COST);
13134   format %{ "ror    $dst, $src, $shift" %}
13135 
13136   ins_encode %{
13137      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13138                $shift$$constant & 0x3f);
13139   %}
13140   ins_pipe(ialu_reg_reg_vshift);
13141 %}
13142 
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13146 %{
13147   match(Set dst (RotateRight src shift));
13148 
13149   ins_cost(INSN_COST);
13150   format %{ "ror    $dst, $src, $shift" %}
13151 
13152   ins_encode %{
13153      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13154   %}
13155   ins_pipe(ialu_reg_reg_vshift);
13156 %}
13157 
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13161 %{
13162   match(Set dst (RotateRight src shift));
13163 
13164   ins_cost(INSN_COST);
13165   format %{ "ror    $dst, $src, $shift" %}
13166 
13167   ins_encode %{
13168      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13169   %}
13170   ins_pipe(ialu_reg_reg_vshift);
13171 %}
13172 
13173 // This pattern is automatically generated from aarch64_ad.m4
13174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13175 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13176 %{
13177   match(Set dst (RotateLeft src shift));
13178 
13179   ins_cost(INSN_COST);
13180   format %{ "rol    $dst, $src, $shift" %}
13181 
13182   ins_encode %{
13183      __ subw(rscratch1, zr, as_Register($shift$$reg));
13184      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13185   %}
13186   ins_pipe(ialu_reg_reg_vshift);
13187 %}
13188 
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13192 %{
13193   match(Set dst (RotateLeft src shift));
13194 
13195   ins_cost(INSN_COST);
13196   format %{ "rol    $dst, $src, $shift" %}
13197 
13198   ins_encode %{
13199      __ subw(rscratch1, zr, as_Register($shift$$reg));
13200      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13201   %}
13202   ins_pipe(ialu_reg_reg_vshift);
13203 %}
13204 
13205 
13206 // Add/subtract (extended)
13207 
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13211 %{
13212   match(Set dst (AddL src1 (ConvI2L src2)));
13213   ins_cost(INSN_COST);
13214   format %{ "add  $dst, $src1, $src2, sxtw" %}
13215 
13216    ins_encode %{
13217      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13218             as_Register($src2$$reg), ext::sxtw);
13219    %}
13220   ins_pipe(ialu_reg_reg);
13221 %}
13222 
13223 // This pattern is automatically generated from aarch64_ad.m4
13224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13225 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13226 %{
13227   match(Set dst (SubL src1 (ConvI2L src2)));
13228   ins_cost(INSN_COST);
13229   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13230 
13231    ins_encode %{
13232      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13233             as_Register($src2$$reg), ext::sxtw);
13234    %}
13235   ins_pipe(ialu_reg_reg);
13236 %}
13237 
13238 // This pattern is automatically generated from aarch64_ad.m4
13239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13240 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13241 %{
13242   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13243   ins_cost(INSN_COST);
13244   format %{ "add  $dst, $src1, $src2, sxth" %}
13245 
13246    ins_encode %{
13247      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13248             as_Register($src2$$reg), ext::sxth);
13249    %}
13250   ins_pipe(ialu_reg_reg);
13251 %}
13252 
13253 // This pattern is automatically generated from aarch64_ad.m4
13254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13255 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13256 %{
13257   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13258   ins_cost(INSN_COST);
13259   format %{ "add  $dst, $src1, $src2, sxtb" %}
13260 
13261    ins_encode %{
13262      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13263             as_Register($src2$$reg), ext::sxtb);
13264    %}
13265   ins_pipe(ialu_reg_reg);
13266 %}
13267 
13268 // This pattern is automatically generated from aarch64_ad.m4
13269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13270 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13271 %{
13272   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13273   ins_cost(INSN_COST);
13274   format %{ "add  $dst, $src1, $src2, uxtb" %}
13275 
13276    ins_encode %{
13277      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13278             as_Register($src2$$reg), ext::uxtb);
13279    %}
13280   ins_pipe(ialu_reg_reg);
13281 %}
13282 
13283 // This pattern is automatically generated from aarch64_ad.m4
13284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13285 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13286 %{
13287   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13288   ins_cost(INSN_COST);
13289   format %{ "add  $dst, $src1, $src2, sxth" %}
13290 
13291    ins_encode %{
13292      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13293             as_Register($src2$$reg), ext::sxth);
13294    %}
13295   ins_pipe(ialu_reg_reg);
13296 %}
13297 
13298 // This pattern is automatically generated from aarch64_ad.m4
13299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13300 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13301 %{
13302   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13303   ins_cost(INSN_COST);
13304   format %{ "add  $dst, $src1, $src2, sxtw" %}
13305 
13306    ins_encode %{
13307      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13308             as_Register($src2$$reg), ext::sxtw);
13309    %}
13310   ins_pipe(ialu_reg_reg);
13311 %}
13312 
13313 // This pattern is automatically generated from aarch64_ad.m4
13314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13315 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13316 %{
13317   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13318   ins_cost(INSN_COST);
13319   format %{ "add  $dst, $src1, $src2, sxtb" %}
13320 
13321    ins_encode %{
13322      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13323             as_Register($src2$$reg), ext::sxtb);
13324    %}
13325   ins_pipe(ialu_reg_reg);
13326 %}
13327 
13328 // This pattern is automatically generated from aarch64_ad.m4
13329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13330 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13331 %{
13332   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13333   ins_cost(INSN_COST);
13334   format %{ "add  $dst, $src1, $src2, uxtb" %}
13335 
13336    ins_encode %{
13337      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13338             as_Register($src2$$reg), ext::uxtb);
13339    %}
13340   ins_pipe(ialu_reg_reg);
13341 %}
13342 
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13346 %{
13347   match(Set dst (AddI src1 (AndI src2 mask)));
13348   ins_cost(INSN_COST);
13349   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13350 
13351    ins_encode %{
13352      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13353             as_Register($src2$$reg), ext::uxtb);
13354    %}
13355   ins_pipe(ialu_reg_reg);
13356 %}
13357 
13358 // This pattern is automatically generated from aarch64_ad.m4
13359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13360 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13361 %{
13362   match(Set dst (AddI src1 (AndI src2 mask)));
13363   ins_cost(INSN_COST);
13364   format %{ "addw  $dst, $src1, $src2, uxth" %}
13365 
13366    ins_encode %{
13367      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13368             as_Register($src2$$reg), ext::uxth);
13369    %}
13370   ins_pipe(ialu_reg_reg);
13371 %}
13372 
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13376 %{
13377   match(Set dst (AddL src1 (AndL src2 mask)));
13378   ins_cost(INSN_COST);
13379   format %{ "add  $dst, $src1, $src2, uxtb" %}
13380 
13381    ins_encode %{
13382      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13383             as_Register($src2$$reg), ext::uxtb);
13384    %}
13385   ins_pipe(ialu_reg_reg);
13386 %}
13387 
13388 // This pattern is automatically generated from aarch64_ad.m4
13389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13390 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13391 %{
13392   match(Set dst (AddL src1 (AndL src2 mask)));
13393   ins_cost(INSN_COST);
13394   format %{ "add  $dst, $src1, $src2, uxth" %}
13395 
13396    ins_encode %{
13397      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13398             as_Register($src2$$reg), ext::uxth);
13399    %}
13400   ins_pipe(ialu_reg_reg);
13401 %}
13402 
13403 // This pattern is automatically generated from aarch64_ad.m4
13404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13405 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13406 %{
13407   match(Set dst (AddL src1 (AndL src2 mask)));
13408   ins_cost(INSN_COST);
13409   format %{ "add  $dst, $src1, $src2, uxtw" %}
13410 
13411    ins_encode %{
13412      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13413             as_Register($src2$$reg), ext::uxtw);
13414    %}
13415   ins_pipe(ialu_reg_reg);
13416 %}
13417 
13418 // This pattern is automatically generated from aarch64_ad.m4
13419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13420 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13421 %{
13422   match(Set dst (SubI src1 (AndI src2 mask)));
13423   ins_cost(INSN_COST);
13424   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13425 
13426    ins_encode %{
13427      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13428             as_Register($src2$$reg), ext::uxtb);
13429    %}
13430   ins_pipe(ialu_reg_reg);
13431 %}
13432 
13433 // This pattern is automatically generated from aarch64_ad.m4
13434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13435 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13436 %{
13437   match(Set dst (SubI src1 (AndI src2 mask)));
13438   ins_cost(INSN_COST);
13439   format %{ "subw  $dst, $src1, $src2, uxth" %}
13440 
13441    ins_encode %{
13442      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13443             as_Register($src2$$reg), ext::uxth);
13444    %}
13445   ins_pipe(ialu_reg_reg);
13446 %}
13447 
13448 // This pattern is automatically generated from aarch64_ad.m4
13449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13450 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13451 %{
13452   match(Set dst (SubL src1 (AndL src2 mask)));
13453   ins_cost(INSN_COST);
13454   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13455 
13456    ins_encode %{
13457      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13458             as_Register($src2$$reg), ext::uxtb);
13459    %}
13460   ins_pipe(ialu_reg_reg);
13461 %}
13462 
13463 // This pattern is automatically generated from aarch64_ad.m4
13464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13465 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13466 %{
13467   match(Set dst (SubL src1 (AndL src2 mask)));
13468   ins_cost(INSN_COST);
13469   format %{ "sub  $dst, $src1, $src2, uxth" %}
13470 
13471    ins_encode %{
13472      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13473             as_Register($src2$$reg), ext::uxth);
13474    %}
13475   ins_pipe(ialu_reg_reg);
13476 %}
13477 
13478 // This pattern is automatically generated from aarch64_ad.m4
13479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13480 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13481 %{
13482   match(Set dst (SubL src1 (AndL src2 mask)));
13483   ins_cost(INSN_COST);
13484   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13485 
13486    ins_encode %{
13487      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13488             as_Register($src2$$reg), ext::uxtw);
13489    %}
13490   ins_pipe(ialu_reg_reg);
13491 %}
13492 
13493 
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13497 %{
13498   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13499   ins_cost(1.9 * INSN_COST);
13500   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13501 
13502    ins_encode %{
13503      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13504             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13505    %}
13506   ins_pipe(ialu_reg_reg_shift);
13507 %}
13508 
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13512 %{
13513   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13514   ins_cost(1.9 * INSN_COST);
13515   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13516 
13517    ins_encode %{
13518      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13519             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13520    %}
13521   ins_pipe(ialu_reg_reg_shift);
13522 %}
13523 
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13527 %{
13528   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13529   ins_cost(1.9 * INSN_COST);
13530   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13531 
13532    ins_encode %{
13533      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13534             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13535    %}
13536   ins_pipe(ialu_reg_reg_shift);
13537 %}
13538 
13539 // This pattern is automatically generated from aarch64_ad.m4
13540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13541 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13542 %{
13543   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13544   ins_cost(1.9 * INSN_COST);
13545   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13546 
13547    ins_encode %{
13548      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13549             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13550    %}
13551   ins_pipe(ialu_reg_reg_shift);
13552 %}
13553 
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13557 %{
13558   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13559   ins_cost(1.9 * INSN_COST);
13560   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13561 
13562    ins_encode %{
13563      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13564             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13565    %}
13566   ins_pipe(ialu_reg_reg_shift);
13567 %}
13568 
13569 // This pattern is automatically generated from aarch64_ad.m4
13570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13571 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13572 %{
13573   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13574   ins_cost(1.9 * INSN_COST);
13575   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13576 
13577    ins_encode %{
13578      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13579             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13580    %}
13581   ins_pipe(ialu_reg_reg_shift);
13582 %}
13583 
13584 // This pattern is automatically generated from aarch64_ad.m4
13585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13586 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13587 %{
13588   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13589   ins_cost(1.9 * INSN_COST);
13590   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13591 
13592    ins_encode %{
13593      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13594             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13595    %}
13596   ins_pipe(ialu_reg_reg_shift);
13597 %}
13598 
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13602 %{
13603   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13604   ins_cost(1.9 * INSN_COST);
13605   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13606 
13607    ins_encode %{
13608      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13609             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13610    %}
13611   ins_pipe(ialu_reg_reg_shift);
13612 %}
13613 
13614 // This pattern is automatically generated from aarch64_ad.m4
13615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13616 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13617 %{
13618   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13619   ins_cost(1.9 * INSN_COST);
13620   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13621 
13622    ins_encode %{
13623      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13624             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13625    %}
13626   ins_pipe(ialu_reg_reg_shift);
13627 %}
13628 
13629 // This pattern is automatically generated from aarch64_ad.m4
13630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13631 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13632 %{
13633   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13634   ins_cost(1.9 * INSN_COST);
13635   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13636 
13637    ins_encode %{
13638      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13639             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13640    %}
13641   ins_pipe(ialu_reg_reg_shift);
13642 %}
13643 
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13647 %{
13648   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13649   ins_cost(1.9 * INSN_COST);
13650   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13651 
13652    ins_encode %{
13653      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13654             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13655    %}
13656   ins_pipe(ialu_reg_reg_shift);
13657 %}
13658 
13659 // This pattern is automatically generated from aarch64_ad.m4
13660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13661 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13662 %{
13663   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13664   ins_cost(1.9 * INSN_COST);
13665   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13666 
13667    ins_encode %{
13668      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13669             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13670    %}
13671   ins_pipe(ialu_reg_reg_shift);
13672 %}
13673 
13674 // This pattern is automatically generated from aarch64_ad.m4
13675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13676 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13677 %{
13678   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13679   ins_cost(1.9 * INSN_COST);
13680   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13681 
13682    ins_encode %{
13683      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13684             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13685    %}
13686   ins_pipe(ialu_reg_reg_shift);
13687 %}
13688 
13689 // This pattern is automatically generated from aarch64_ad.m4
13690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13691 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13692 %{
13693   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13694   ins_cost(1.9 * INSN_COST);
13695   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13696 
13697    ins_encode %{
13698      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13699             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13700    %}
13701   ins_pipe(ialu_reg_reg_shift);
13702 %}
13703 
13704 // This pattern is automatically generated from aarch64_ad.m4
13705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13706 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13707 %{
13708   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13709   ins_cost(1.9 * INSN_COST);
13710   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13711 
13712    ins_encode %{
13713      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13714             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13715    %}
13716   ins_pipe(ialu_reg_reg_shift);
13717 %}
13718 
13719 // This pattern is automatically generated from aarch64_ad.m4
13720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13721 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13722 %{
13723   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13724   ins_cost(1.9 * INSN_COST);
13725   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13726 
13727    ins_encode %{
13728      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13729             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13730    %}
13731   ins_pipe(ialu_reg_reg_shift);
13732 %}
13733 
13734 // This pattern is automatically generated from aarch64_ad.m4
13735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13736 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13737 %{
13738   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13739   ins_cost(1.9 * INSN_COST);
13740   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13741 
13742    ins_encode %{
13743      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13744             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13745    %}
13746   ins_pipe(ialu_reg_reg_shift);
13747 %}
13748 
13749 // This pattern is automatically generated from aarch64_ad.m4
13750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13751 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13752 %{
13753   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13754   ins_cost(1.9 * INSN_COST);
13755   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13756 
13757    ins_encode %{
13758      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13759             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13760    %}
13761   ins_pipe(ialu_reg_reg_shift);
13762 %}
13763 
13764 // This pattern is automatically generated from aarch64_ad.m4
13765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13766 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13767 %{
13768   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13769   ins_cost(1.9 * INSN_COST);
13770   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13771 
13772    ins_encode %{
13773      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13774             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13775    %}
13776   ins_pipe(ialu_reg_reg_shift);
13777 %}
13778 
13779 // This pattern is automatically generated from aarch64_ad.m4
13780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13781 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13782 %{
13783   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13784   ins_cost(1.9 * INSN_COST);
13785   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13786 
13787    ins_encode %{
13788      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13789             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13790    %}
13791   ins_pipe(ialu_reg_reg_shift);
13792 %}
13793 
13794 // This pattern is automatically generated from aarch64_ad.m4
13795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13796 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13797 %{
13798   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13799   ins_cost(1.9 * INSN_COST);
13800   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13801 
13802    ins_encode %{
13803      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13804             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13805    %}
13806   ins_pipe(ialu_reg_reg_shift);
13807 %}
13808 
13809 // This pattern is automatically generated from aarch64_ad.m4
13810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13811 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13812 %{
13813   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13814   ins_cost(1.9 * INSN_COST);
13815   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13816 
13817    ins_encode %{
13818      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13819             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13820    %}
13821   ins_pipe(ialu_reg_reg_shift);
13822 %}
13823 
13824 // This pattern is automatically generated from aarch64_ad.m4
13825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13826 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13827 %{
13828   effect(DEF dst, USE src1, USE src2, USE cr);
13829   ins_cost(INSN_COST * 2);
13830   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13831 
13832   ins_encode %{
13833     __ cselw($dst$$Register,
13834              $src1$$Register,
13835              $src2$$Register,
13836              Assembler::LT);
13837   %}
13838   ins_pipe(icond_reg_reg);
13839 %}
13840 
13841 // This pattern is automatically generated from aarch64_ad.m4
13842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13843 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13844 %{
13845   effect(DEF dst, USE src1, USE src2, USE cr);
13846   ins_cost(INSN_COST * 2);
13847   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13848 
13849   ins_encode %{
13850     __ cselw($dst$$Register,
13851              $src1$$Register,
13852              $src2$$Register,
13853              Assembler::GT);
13854   %}
13855   ins_pipe(icond_reg_reg);
13856 %}
13857 
13858 // This pattern is automatically generated from aarch64_ad.m4
13859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13860 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13861 %{
13862   effect(DEF dst, USE src1, USE cr);
13863   ins_cost(INSN_COST * 2);
13864   format %{ "cselw $dst, $src1, zr lt\t"  %}
13865 
13866   ins_encode %{
13867     __ cselw($dst$$Register,
13868              $src1$$Register,
13869              zr,
13870              Assembler::LT);
13871   %}
13872   ins_pipe(icond_reg);
13873 %}
13874 
13875 // This pattern is automatically generated from aarch64_ad.m4
13876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13877 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13878 %{
13879   effect(DEF dst, USE src1, USE cr);
13880   ins_cost(INSN_COST * 2);
13881   format %{ "cselw $dst, $src1, zr gt\t"  %}
13882 
13883   ins_encode %{
13884     __ cselw($dst$$Register,
13885              $src1$$Register,
13886              zr,
13887              Assembler::GT);
13888   %}
13889   ins_pipe(icond_reg);
13890 %}
13891 
13892 // This pattern is automatically generated from aarch64_ad.m4
13893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13894 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13895 %{
13896   effect(DEF dst, USE src1, USE cr);
13897   ins_cost(INSN_COST * 2);
13898   format %{ "csincw $dst, $src1, zr le\t"  %}
13899 
13900   ins_encode %{
13901     __ csincw($dst$$Register,
13902              $src1$$Register,
13903              zr,
13904              Assembler::LE);
13905   %}
13906   ins_pipe(icond_reg);
13907 %}
13908 
13909 // This pattern is automatically generated from aarch64_ad.m4
13910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13911 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13912 %{
13913   effect(DEF dst, USE src1, USE cr);
13914   ins_cost(INSN_COST * 2);
13915   format %{ "csincw $dst, $src1, zr gt\t"  %}
13916 
13917   ins_encode %{
13918     __ csincw($dst$$Register,
13919              $src1$$Register,
13920              zr,
13921              Assembler::GT);
13922   %}
13923   ins_pipe(icond_reg);
13924 %}
13925 
13926 // This pattern is automatically generated from aarch64_ad.m4
13927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13928 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13929 %{
13930   effect(DEF dst, USE src1, USE cr);
13931   ins_cost(INSN_COST * 2);
13932   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13933 
13934   ins_encode %{
13935     __ csinvw($dst$$Register,
13936              $src1$$Register,
13937              zr,
13938              Assembler::LT);
13939   %}
13940   ins_pipe(icond_reg);
13941 %}
13942 
13943 // This pattern is automatically generated from aarch64_ad.m4
13944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13945 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13946 %{
13947   effect(DEF dst, USE src1, USE cr);
13948   ins_cost(INSN_COST * 2);
13949   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13950 
13951   ins_encode %{
13952     __ csinvw($dst$$Register,
13953              $src1$$Register,
13954              zr,
13955              Assembler::GE);
13956   %}
13957   ins_pipe(icond_reg);
13958 %}
13959 
13960 // This pattern is automatically generated from aarch64_ad.m4
13961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13962 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13963 %{
13964   match(Set dst (MinI src imm));
13965   ins_cost(INSN_COST * 3);
13966   expand %{
13967     rFlagsReg cr;
13968     compI_reg_imm0(cr, src);
13969     cmovI_reg_imm0_lt(dst, src, cr);
13970   %}
13971 %}
13972 
13973 // This pattern is automatically generated from aarch64_ad.m4
13974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13975 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13976 %{
13977   match(Set dst (MinI imm src));
13978   ins_cost(INSN_COST * 3);
13979   expand %{
13980     rFlagsReg cr;
13981     compI_reg_imm0(cr, src);
13982     cmovI_reg_imm0_lt(dst, src, cr);
13983   %}
13984 %}
13985 
13986 // This pattern is automatically generated from aarch64_ad.m4
13987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13988 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13989 %{
13990   match(Set dst (MinI src imm));
13991   ins_cost(INSN_COST * 3);
13992   expand %{
13993     rFlagsReg cr;
13994     compI_reg_imm0(cr, src);
13995     cmovI_reg_imm1_le(dst, src, cr);
13996   %}
13997 %}
13998 
13999 // This pattern is automatically generated from aarch64_ad.m4
14000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14001 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14002 %{
14003   match(Set dst (MinI imm src));
14004   ins_cost(INSN_COST * 3);
14005   expand %{
14006     rFlagsReg cr;
14007     compI_reg_imm0(cr, src);
14008     cmovI_reg_imm1_le(dst, src, cr);
14009   %}
14010 %}
14011 
14012 // This pattern is automatically generated from aarch64_ad.m4
14013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14014 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14015 %{
14016   match(Set dst (MinI src imm));
14017   ins_cost(INSN_COST * 3);
14018   expand %{
14019     rFlagsReg cr;
14020     compI_reg_imm0(cr, src);
14021     cmovI_reg_immM1_lt(dst, src, cr);
14022   %}
14023 %}
14024 
14025 // This pattern is automatically generated from aarch64_ad.m4
14026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14027 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14028 %{
14029   match(Set dst (MinI imm src));
14030   ins_cost(INSN_COST * 3);
14031   expand %{
14032     rFlagsReg cr;
14033     compI_reg_imm0(cr, src);
14034     cmovI_reg_immM1_lt(dst, src, cr);
14035   %}
14036 %}
14037 
14038 // This pattern is automatically generated from aarch64_ad.m4
14039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14040 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14041 %{
14042   match(Set dst (MaxI src imm));
14043   ins_cost(INSN_COST * 3);
14044   expand %{
14045     rFlagsReg cr;
14046     compI_reg_imm0(cr, src);
14047     cmovI_reg_imm0_gt(dst, src, cr);
14048   %}
14049 %}
14050 
14051 // This pattern is automatically generated from aarch64_ad.m4
14052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14053 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14054 %{
14055   match(Set dst (MaxI imm src));
14056   ins_cost(INSN_COST * 3);
14057   expand %{
14058     rFlagsReg cr;
14059     compI_reg_imm0(cr, src);
14060     cmovI_reg_imm0_gt(dst, src, cr);
14061   %}
14062 %}
14063 
14064 // This pattern is automatically generated from aarch64_ad.m4
14065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14066 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14067 %{
14068   match(Set dst (MaxI src imm));
14069   ins_cost(INSN_COST * 3);
14070   expand %{
14071     rFlagsReg cr;
14072     compI_reg_imm0(cr, src);
14073     cmovI_reg_imm1_gt(dst, src, cr);
14074   %}
14075 %}
14076 
14077 // This pattern is automatically generated from aarch64_ad.m4
14078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14079 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14080 %{
14081   match(Set dst (MaxI imm src));
14082   ins_cost(INSN_COST * 3);
14083   expand %{
14084     rFlagsReg cr;
14085     compI_reg_imm0(cr, src);
14086     cmovI_reg_imm1_gt(dst, src, cr);
14087   %}
14088 %}
14089 
14090 // This pattern is automatically generated from aarch64_ad.m4
14091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14092 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14093 %{
14094   match(Set dst (MaxI src imm));
14095   ins_cost(INSN_COST * 3);
14096   expand %{
14097     rFlagsReg cr;
14098     compI_reg_imm0(cr, src);
14099     cmovI_reg_immM1_ge(dst, src, cr);
14100   %}
14101 %}
14102 
14103 // This pattern is automatically generated from aarch64_ad.m4
14104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14105 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14106 %{
14107   match(Set dst (MaxI imm src));
14108   ins_cost(INSN_COST * 3);
14109   expand %{
14110     rFlagsReg cr;
14111     compI_reg_imm0(cr, src);
14112     cmovI_reg_immM1_ge(dst, src, cr);
14113   %}
14114 %}
14115 
14116 
14117 
14118 // END This section of the file is automatically generated. Do not edit --------------
14119 
14120 
14121 // ============================================================================
14122 // Floating Point Arithmetic Instructions
14123 
14124 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14125   match(Set dst (AddF src1 src2));
14126 
14127   ins_cost(INSN_COST * 5);
14128   format %{ "fadds   $dst, $src1, $src2" %}
14129 
14130   ins_encode %{
14131     __ fadds(as_FloatRegister($dst$$reg),
14132              as_FloatRegister($src1$$reg),
14133              as_FloatRegister($src2$$reg));
14134   %}
14135 
14136   ins_pipe(fp_dop_reg_reg_s);
14137 %}
14138 
14139 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14140   match(Set dst (AddD src1 src2));
14141 
14142   ins_cost(INSN_COST * 5);
14143   format %{ "faddd   $dst, $src1, $src2" %}
14144 
14145   ins_encode %{
14146     __ faddd(as_FloatRegister($dst$$reg),
14147              as_FloatRegister($src1$$reg),
14148              as_FloatRegister($src2$$reg));
14149   %}
14150 
14151   ins_pipe(fp_dop_reg_reg_d);
14152 %}
14153 
14154 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14155   match(Set dst (SubF src1 src2));
14156 
14157   ins_cost(INSN_COST * 5);
14158   format %{ "fsubs   $dst, $src1, $src2" %}
14159 
14160   ins_encode %{
14161     __ fsubs(as_FloatRegister($dst$$reg),
14162              as_FloatRegister($src1$$reg),
14163              as_FloatRegister($src2$$reg));
14164   %}
14165 
14166   ins_pipe(fp_dop_reg_reg_s);
14167 %}
14168 
14169 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14170   match(Set dst (SubD src1 src2));
14171 
14172   ins_cost(INSN_COST * 5);
14173   format %{ "fsubd   $dst, $src1, $src2" %}
14174 
14175   ins_encode %{
14176     __ fsubd(as_FloatRegister($dst$$reg),
14177              as_FloatRegister($src1$$reg),
14178              as_FloatRegister($src2$$reg));
14179   %}
14180 
14181   ins_pipe(fp_dop_reg_reg_d);
14182 %}
14183 
14184 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14185   match(Set dst (MulF src1 src2));
14186 
14187   ins_cost(INSN_COST * 6);
14188   format %{ "fmuls   $dst, $src1, $src2" %}
14189 
14190   ins_encode %{
14191     __ fmuls(as_FloatRegister($dst$$reg),
14192              as_FloatRegister($src1$$reg),
14193              as_FloatRegister($src2$$reg));
14194   %}
14195 
14196   ins_pipe(fp_dop_reg_reg_s);
14197 %}
14198 
14199 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14200   match(Set dst (MulD src1 src2));
14201 
14202   ins_cost(INSN_COST * 6);
14203   format %{ "fmuld   $dst, $src1, $src2" %}
14204 
14205   ins_encode %{
14206     __ fmuld(as_FloatRegister($dst$$reg),
14207              as_FloatRegister($src1$$reg),
14208              as_FloatRegister($src2$$reg));
14209   %}
14210 
14211   ins_pipe(fp_dop_reg_reg_d);
14212 %}
14213 
14214 // src1 * src2 + src3
14215 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14216   predicate(UseFMA);
14217   match(Set dst (FmaF src3 (Binary src1 src2)));
14218 
14219   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14220 
14221   ins_encode %{
14222     __ fmadds(as_FloatRegister($dst$$reg),
14223              as_FloatRegister($src1$$reg),
14224              as_FloatRegister($src2$$reg),
14225              as_FloatRegister($src3$$reg));
14226   %}
14227 
14228   ins_pipe(pipe_class_default);
14229 %}
14230 
14231 // src1 * src2 + src3
14232 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14233   predicate(UseFMA);
14234   match(Set dst (FmaD src3 (Binary src1 src2)));
14235 
14236   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14237 
14238   ins_encode %{
14239     __ fmaddd(as_FloatRegister($dst$$reg),
14240              as_FloatRegister($src1$$reg),
14241              as_FloatRegister($src2$$reg),
14242              as_FloatRegister($src3$$reg));
14243   %}
14244 
14245   ins_pipe(pipe_class_default);
14246 %}
14247 
14248 // -src1 * src2 + src3
14249 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14250   predicate(UseFMA);
14251   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14252   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14253 
14254   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14255 
14256   ins_encode %{
14257     __ fmsubs(as_FloatRegister($dst$$reg),
14258               as_FloatRegister($src1$$reg),
14259               as_FloatRegister($src2$$reg),
14260               as_FloatRegister($src3$$reg));
14261   %}
14262 
14263   ins_pipe(pipe_class_default);
14264 %}
14265 
14266 // -src1 * src2 + src3
14267 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14268   predicate(UseFMA);
14269   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14270   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14271 
14272   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14273 
14274   ins_encode %{
14275     __ fmsubd(as_FloatRegister($dst$$reg),
14276               as_FloatRegister($src1$$reg),
14277               as_FloatRegister($src2$$reg),
14278               as_FloatRegister($src3$$reg));
14279   %}
14280 
14281   ins_pipe(pipe_class_default);
14282 %}
14283 
14284 // -src1 * src2 - src3
14285 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14286   predicate(UseFMA);
14287   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14288   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14289 
14290   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14291 
14292   ins_encode %{
14293     __ fnmadds(as_FloatRegister($dst$$reg),
14294                as_FloatRegister($src1$$reg),
14295                as_FloatRegister($src2$$reg),
14296                as_FloatRegister($src3$$reg));
14297   %}
14298 
14299   ins_pipe(pipe_class_default);
14300 %}
14301 
14302 // -src1 * src2 - src3
14303 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14304   predicate(UseFMA);
14305   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14306   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14307 
14308   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14309 
14310   ins_encode %{
14311     __ fnmaddd(as_FloatRegister($dst$$reg),
14312                as_FloatRegister($src1$$reg),
14313                as_FloatRegister($src2$$reg),
14314                as_FloatRegister($src3$$reg));
14315   %}
14316 
14317   ins_pipe(pipe_class_default);
14318 %}
14319 
14320 // src1 * src2 - src3
14321 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14322   predicate(UseFMA);
14323   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14324 
14325   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14326 
14327   ins_encode %{
14328     __ fnmsubs(as_FloatRegister($dst$$reg),
14329                as_FloatRegister($src1$$reg),
14330                as_FloatRegister($src2$$reg),
14331                as_FloatRegister($src3$$reg));
14332   %}
14333 
14334   ins_pipe(pipe_class_default);
14335 %}
14336 
14337 // src1 * src2 - src3
14338 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14339   predicate(UseFMA);
14340   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14341 
14342   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14343 
14344   ins_encode %{
14345   // n.b. insn name should be fnmsubd
14346     __ fnmsub(as_FloatRegister($dst$$reg),
14347               as_FloatRegister($src1$$reg),
14348               as_FloatRegister($src2$$reg),
14349               as_FloatRegister($src3$$reg));
14350   %}
14351 
14352   ins_pipe(pipe_class_default);
14353 %}
14354 
14355 
14356 // Math.max(FF)F
14357 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14358   match(Set dst (MaxF src1 src2));
14359 
14360   format %{ "fmaxs   $dst, $src1, $src2" %}
14361   ins_encode %{
14362     __ fmaxs(as_FloatRegister($dst$$reg),
14363              as_FloatRegister($src1$$reg),
14364              as_FloatRegister($src2$$reg));
14365   %}
14366 
14367   ins_pipe(fp_dop_reg_reg_s);
14368 %}
14369 
14370 // Math.min(FF)F
14371 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14372   match(Set dst (MinF src1 src2));
14373 
14374   format %{ "fmins   $dst, $src1, $src2" %}
14375   ins_encode %{
14376     __ fmins(as_FloatRegister($dst$$reg),
14377              as_FloatRegister($src1$$reg),
14378              as_FloatRegister($src2$$reg));
14379   %}
14380 
14381   ins_pipe(fp_dop_reg_reg_s);
14382 %}
14383 
14384 // Math.max(DD)D
14385 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14386   match(Set dst (MaxD src1 src2));
14387 
14388   format %{ "fmaxd   $dst, $src1, $src2" %}
14389   ins_encode %{
14390     __ fmaxd(as_FloatRegister($dst$$reg),
14391              as_FloatRegister($src1$$reg),
14392              as_FloatRegister($src2$$reg));
14393   %}
14394 
14395   ins_pipe(fp_dop_reg_reg_d);
14396 %}
14397 
14398 // Math.min(DD)D
14399 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14400   match(Set dst (MinD src1 src2));
14401 
14402   format %{ "fmind   $dst, $src1, $src2" %}
14403   ins_encode %{
14404     __ fmind(as_FloatRegister($dst$$reg),
14405              as_FloatRegister($src1$$reg),
14406              as_FloatRegister($src2$$reg));
14407   %}
14408 
14409   ins_pipe(fp_dop_reg_reg_d);
14410 %}
14411 
14412 
14413 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14414   match(Set dst (DivF src1  src2));
14415 
14416   ins_cost(INSN_COST * 18);
14417   format %{ "fdivs   $dst, $src1, $src2" %}
14418 
14419   ins_encode %{
14420     __ fdivs(as_FloatRegister($dst$$reg),
14421              as_FloatRegister($src1$$reg),
14422              as_FloatRegister($src2$$reg));
14423   %}
14424 
14425   ins_pipe(fp_div_s);
14426 %}
14427 
14428 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14429   match(Set dst (DivD src1  src2));
14430 
14431   ins_cost(INSN_COST * 32);
14432   format %{ "fdivd   $dst, $src1, $src2" %}
14433 
14434   ins_encode %{
14435     __ fdivd(as_FloatRegister($dst$$reg),
14436              as_FloatRegister($src1$$reg),
14437              as_FloatRegister($src2$$reg));
14438   %}
14439 
14440   ins_pipe(fp_div_d);
14441 %}
14442 
14443 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14444   match(Set dst (NegF src));
14445 
14446   ins_cost(INSN_COST * 3);
14447   format %{ "fneg   $dst, $src" %}
14448 
14449   ins_encode %{
14450     __ fnegs(as_FloatRegister($dst$$reg),
14451              as_FloatRegister($src$$reg));
14452   %}
14453 
14454   ins_pipe(fp_uop_s);
14455 %}
14456 
14457 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14458   match(Set dst (NegD src));
14459 
14460   ins_cost(INSN_COST * 3);
14461   format %{ "fnegd   $dst, $src" %}
14462 
14463   ins_encode %{
14464     __ fnegd(as_FloatRegister($dst$$reg),
14465              as_FloatRegister($src$$reg));
14466   %}
14467 
14468   ins_pipe(fp_uop_d);
14469 %}
14470 
14471 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14472 %{
14473   match(Set dst (AbsI src));
14474 
14475   effect(KILL cr);
14476   ins_cost(INSN_COST * 2);
14477   format %{ "cmpw  $src, zr\n\t"
14478             "cnegw $dst, $src, Assembler::LT\t# int abs"
14479   %}
14480 
14481   ins_encode %{
14482     __ cmpw(as_Register($src$$reg), zr);
14483     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14484   %}
14485   ins_pipe(pipe_class_default);
14486 %}
14487 
14488 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14489 %{
14490   match(Set dst (AbsL src));
14491 
14492   effect(KILL cr);
14493   ins_cost(INSN_COST * 2);
14494   format %{ "cmp  $src, zr\n\t"
14495             "cneg $dst, $src, Assembler::LT\t# long abs"
14496   %}
14497 
14498   ins_encode %{
14499     __ cmp(as_Register($src$$reg), zr);
14500     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14501   %}
14502   ins_pipe(pipe_class_default);
14503 %}
14504 
14505 instruct absF_reg(vRegF dst, vRegF src) %{
14506   match(Set dst (AbsF src));
14507 
14508   ins_cost(INSN_COST * 3);
14509   format %{ "fabss   $dst, $src" %}
14510   ins_encode %{
14511     __ fabss(as_FloatRegister($dst$$reg),
14512              as_FloatRegister($src$$reg));
14513   %}
14514 
14515   ins_pipe(fp_uop_s);
14516 %}
14517 
14518 instruct absD_reg(vRegD dst, vRegD src) %{
14519   match(Set dst (AbsD src));
14520 
14521   ins_cost(INSN_COST * 3);
14522   format %{ "fabsd   $dst, $src" %}
14523   ins_encode %{
14524     __ fabsd(as_FloatRegister($dst$$reg),
14525              as_FloatRegister($src$$reg));
14526   %}
14527 
14528   ins_pipe(fp_uop_d);
14529 %}
14530 
14531 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14532   match(Set dst (AbsF (SubF src1 src2)));
14533 
14534   ins_cost(INSN_COST * 3);
14535   format %{ "fabds   $dst, $src1, $src2" %}
14536   ins_encode %{
14537     __ fabds(as_FloatRegister($dst$$reg),
14538              as_FloatRegister($src1$$reg),
14539              as_FloatRegister($src2$$reg));
14540   %}
14541 
14542   ins_pipe(fp_uop_s);
14543 %}
14544 
14545 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14546   match(Set dst (AbsD (SubD src1 src2)));
14547 
14548   ins_cost(INSN_COST * 3);
14549   format %{ "fabdd   $dst, $src1, $src2" %}
14550   ins_encode %{
14551     __ fabdd(as_FloatRegister($dst$$reg),
14552              as_FloatRegister($src1$$reg),
14553              as_FloatRegister($src2$$reg));
14554   %}
14555 
14556   ins_pipe(fp_uop_d);
14557 %}
14558 
14559 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14560   match(Set dst (SqrtD src));
14561 
14562   ins_cost(INSN_COST * 50);
14563   format %{ "fsqrtd  $dst, $src" %}
14564   ins_encode %{
14565     __ fsqrtd(as_FloatRegister($dst$$reg),
14566              as_FloatRegister($src$$reg));
14567   %}
14568 
14569   ins_pipe(fp_div_s);
14570 %}
14571 
14572 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14573   match(Set dst (SqrtF src));
14574 
14575   ins_cost(INSN_COST * 50);
14576   format %{ "fsqrts  $dst, $src" %}
14577   ins_encode %{
14578     __ fsqrts(as_FloatRegister($dst$$reg),
14579              as_FloatRegister($src$$reg));
14580   %}
14581 
14582   ins_pipe(fp_div_d);
14583 %}
14584 
14585 // Math.rint, floor, ceil
14586 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14587   match(Set dst (RoundDoubleMode src rmode));
14588   format %{ "frint  $dst, $src, $rmode" %}
14589   ins_encode %{
14590     switch ($rmode$$constant) {
14591       case RoundDoubleModeNode::rmode_rint:
14592         __ frintnd(as_FloatRegister($dst$$reg),
14593                    as_FloatRegister($src$$reg));
14594         break;
14595       case RoundDoubleModeNode::rmode_floor:
14596         __ frintmd(as_FloatRegister($dst$$reg),
14597                    as_FloatRegister($src$$reg));
14598         break;
14599       case RoundDoubleModeNode::rmode_ceil:
14600         __ frintpd(as_FloatRegister($dst$$reg),
14601                    as_FloatRegister($src$$reg));
14602         break;
14603     }
14604   %}
14605   ins_pipe(fp_uop_d);
14606 %}
14607 
14608 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14609   match(Set dst (CopySignD src1 (Binary src2 zero)));
14610   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14611   format %{ "CopySignD  $dst $src1 $src2" %}
14612   ins_encode %{
14613     FloatRegister dst = as_FloatRegister($dst$$reg),
14614                   src1 = as_FloatRegister($src1$$reg),
14615                   src2 = as_FloatRegister($src2$$reg),
14616                   zero = as_FloatRegister($zero$$reg);
14617     __ fnegd(dst, zero);
14618     __ bsl(dst, __ T8B, src2, src1);
14619   %}
14620   ins_pipe(fp_uop_d);
14621 %}
14622 
14623 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14624   match(Set dst (CopySignF src1 src2));
14625   effect(TEMP_DEF dst, USE src1, USE src2);
14626   format %{ "CopySignF  $dst $src1 $src2" %}
14627   ins_encode %{
14628     FloatRegister dst = as_FloatRegister($dst$$reg),
14629                   src1 = as_FloatRegister($src1$$reg),
14630                   src2 = as_FloatRegister($src2$$reg);
14631     __ movi(dst, __ T2S, 0x80, 24);
14632     __ bsl(dst, __ T8B, src2, src1);
14633   %}
14634   ins_pipe(fp_uop_d);
14635 %}
14636 
14637 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14638   match(Set dst (SignumD src (Binary zero one)));
14639   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14640   format %{ "signumD  $dst, $src" %}
14641   ins_encode %{
14642     FloatRegister src = as_FloatRegister($src$$reg),
14643                   dst = as_FloatRegister($dst$$reg),
14644                   zero = as_FloatRegister($zero$$reg),
14645                   one = as_FloatRegister($one$$reg);
14646     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14647     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14648     // Bit selection instruction gets bit from "one" for each enabled bit in
14649     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14650     // NaN the whole "src" will be copied because "dst" is zero. For all other
14651     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14652     // from "src", and all other bits are copied from 1.0.
14653     __ bsl(dst, __ T8B, one, src);
14654   %}
14655   ins_pipe(fp_uop_d);
14656 %}
14657 
14658 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14659   match(Set dst (SignumF src (Binary zero one)));
14660   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14661   format %{ "signumF  $dst, $src" %}
14662   ins_encode %{
14663     FloatRegister src = as_FloatRegister($src$$reg),
14664                   dst = as_FloatRegister($dst$$reg),
14665                   zero = as_FloatRegister($zero$$reg),
14666                   one = as_FloatRegister($one$$reg);
14667     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14668     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14669     // Bit selection instruction gets bit from "one" for each enabled bit in
14670     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14671     // NaN the whole "src" will be copied because "dst" is zero. For all other
14672     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14673     // from "src", and all other bits are copied from 1.0.
14674     __ bsl(dst, __ T8B, one, src);
14675   %}
14676   ins_pipe(fp_uop_d);
14677 %}
14678 
14679 instruct onspinwait() %{
14680   match(OnSpinWait);
14681   ins_cost(INSN_COST);
14682 
14683   format %{ "onspinwait" %}
14684 
14685   ins_encode %{
14686     __ spin_wait();
14687   %}
14688   ins_pipe(pipe_class_empty);
14689 %}
14690 
14691 // ============================================================================
14692 // Logical Instructions
14693 
14694 // Integer Logical Instructions
14695 
14696 // And Instructions
14697 
14698 
14699 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14700   match(Set dst (AndI src1 src2));
14701 
14702   format %{ "andw  $dst, $src1, $src2\t# int" %}
14703 
14704   ins_cost(INSN_COST);
14705   ins_encode %{
14706     __ andw(as_Register($dst$$reg),
14707             as_Register($src1$$reg),
14708             as_Register($src2$$reg));
14709   %}
14710 
14711   ins_pipe(ialu_reg_reg);
14712 %}
14713 
14714 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14715   match(Set dst (AndI src1 src2));
14716 
14717   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14718 
14719   ins_cost(INSN_COST);
14720   ins_encode %{
14721     __ andw(as_Register($dst$$reg),
14722             as_Register($src1$$reg),
14723             (uint64_t)($src2$$constant));
14724   %}
14725 
14726   ins_pipe(ialu_reg_imm);
14727 %}
14728 
14729 // Or Instructions
14730 
14731 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14732   match(Set dst (OrI src1 src2));
14733 
14734   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14735 
14736   ins_cost(INSN_COST);
14737   ins_encode %{
14738     __ orrw(as_Register($dst$$reg),
14739             as_Register($src1$$reg),
14740             as_Register($src2$$reg));
14741   %}
14742 
14743   ins_pipe(ialu_reg_reg);
14744 %}
14745 
14746 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14747   match(Set dst (OrI src1 src2));
14748 
14749   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14750 
14751   ins_cost(INSN_COST);
14752   ins_encode %{
14753     __ orrw(as_Register($dst$$reg),
14754             as_Register($src1$$reg),
14755             (uint64_t)($src2$$constant));
14756   %}
14757 
14758   ins_pipe(ialu_reg_imm);
14759 %}
14760 
14761 // Xor Instructions
14762 
14763 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14764   match(Set dst (XorI src1 src2));
14765 
14766   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14767 
14768   ins_cost(INSN_COST);
14769   ins_encode %{
14770     __ eorw(as_Register($dst$$reg),
14771             as_Register($src1$$reg),
14772             as_Register($src2$$reg));
14773   %}
14774 
14775   ins_pipe(ialu_reg_reg);
14776 %}
14777 
14778 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14779   match(Set dst (XorI src1 src2));
14780 
14781   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14782 
14783   ins_cost(INSN_COST);
14784   ins_encode %{
14785     __ eorw(as_Register($dst$$reg),
14786             as_Register($src1$$reg),
14787             (uint64_t)($src2$$constant));
14788   %}
14789 
14790   ins_pipe(ialu_reg_imm);
14791 %}
14792 
14793 // Long Logical Instructions
14794 // TODO
14795 
14796 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14797   match(Set dst (AndL src1 src2));
14798 
14799   format %{ "and  $dst, $src1, $src2\t# int" %}
14800 
14801   ins_cost(INSN_COST);
14802   ins_encode %{
14803     __ andr(as_Register($dst$$reg),
14804             as_Register($src1$$reg),
14805             as_Register($src2$$reg));
14806   %}
14807 
14808   ins_pipe(ialu_reg_reg);
14809 %}
14810 
14811 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14812   match(Set dst (AndL src1 src2));
14813 
14814   format %{ "and  $dst, $src1, $src2\t# int" %}
14815 
14816   ins_cost(INSN_COST);
14817   ins_encode %{
14818     __ andr(as_Register($dst$$reg),
14819             as_Register($src1$$reg),
14820             (uint64_t)($src2$$constant));
14821   %}
14822 
14823   ins_pipe(ialu_reg_imm);
14824 %}
14825 
14826 // Or Instructions
14827 
14828 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14829   match(Set dst (OrL src1 src2));
14830 
14831   format %{ "orr  $dst, $src1, $src2\t# int" %}
14832 
14833   ins_cost(INSN_COST);
14834   ins_encode %{
14835     __ orr(as_Register($dst$$reg),
14836            as_Register($src1$$reg),
14837            as_Register($src2$$reg));
14838   %}
14839 
14840   ins_pipe(ialu_reg_reg);
14841 %}
14842 
14843 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14844   match(Set dst (OrL src1 src2));
14845 
14846   format %{ "orr  $dst, $src1, $src2\t# int" %}
14847 
14848   ins_cost(INSN_COST);
14849   ins_encode %{
14850     __ orr(as_Register($dst$$reg),
14851            as_Register($src1$$reg),
14852            (uint64_t)($src2$$constant));
14853   %}
14854 
14855   ins_pipe(ialu_reg_imm);
14856 %}
14857 
14858 // Xor Instructions
14859 
14860 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14861   match(Set dst (XorL src1 src2));
14862 
14863   format %{ "eor  $dst, $src1, $src2\t# int" %}
14864 
14865   ins_cost(INSN_COST);
14866   ins_encode %{
14867     __ eor(as_Register($dst$$reg),
14868            as_Register($src1$$reg),
14869            as_Register($src2$$reg));
14870   %}
14871 
14872   ins_pipe(ialu_reg_reg);
14873 %}
14874 
14875 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14876   match(Set dst (XorL src1 src2));
14877 
14878   ins_cost(INSN_COST);
14879   format %{ "eor  $dst, $src1, $src2\t# int" %}
14880 
14881   ins_encode %{
14882     __ eor(as_Register($dst$$reg),
14883            as_Register($src1$$reg),
14884            (uint64_t)($src2$$constant));
14885   %}
14886 
14887   ins_pipe(ialu_reg_imm);
14888 %}
14889 
14890 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14891 %{
14892   match(Set dst (ConvI2L src));
14893 
14894   ins_cost(INSN_COST);
14895   format %{ "sxtw  $dst, $src\t# i2l" %}
14896   ins_encode %{
14897     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14898   %}
14899   ins_pipe(ialu_reg_shift);
14900 %}
14901 
14902 // this pattern occurs in bigmath arithmetic
14903 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14904 %{
14905   match(Set dst (AndL (ConvI2L src) mask));
14906 
14907   ins_cost(INSN_COST);
14908   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14909   ins_encode %{
14910     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14911   %}
14912 
14913   ins_pipe(ialu_reg_shift);
14914 %}
14915 
14916 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14917   match(Set dst (ConvL2I src));
14918 
14919   ins_cost(INSN_COST);
14920   format %{ "movw  $dst, $src \t// l2i" %}
14921 
14922   ins_encode %{
14923     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14924   %}
14925 
14926   ins_pipe(ialu_reg);
14927 %}
14928 
14929 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14930 %{
14931   match(Set dst (Conv2B src));
14932   effect(KILL cr);
14933 
14934   format %{
14935     "cmpw $src, zr\n\t"
14936     "cset $dst, ne"
14937   %}
14938 
14939   ins_encode %{
14940     __ cmpw(as_Register($src$$reg), zr);
14941     __ cset(as_Register($dst$$reg), Assembler::NE);
14942   %}
14943 
14944   ins_pipe(ialu_reg);
14945 %}
14946 
14947 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14948 %{
14949   match(Set dst (Conv2B src));
14950   effect(KILL cr);
14951 
14952   format %{
14953     "cmp  $src, zr\n\t"
14954     "cset $dst, ne"
14955   %}
14956 
14957   ins_encode %{
14958     __ cmp(as_Register($src$$reg), zr);
14959     __ cset(as_Register($dst$$reg), Assembler::NE);
14960   %}
14961 
14962   ins_pipe(ialu_reg);
14963 %}
14964 
14965 instruct convD2F_reg(vRegF dst, vRegD src) %{
14966   match(Set dst (ConvD2F src));
14967 
14968   ins_cost(INSN_COST * 5);
14969   format %{ "fcvtd  $dst, $src \t// d2f" %}
14970 
14971   ins_encode %{
14972     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14973   %}
14974 
14975   ins_pipe(fp_d2f);
14976 %}
14977 
14978 instruct convF2D_reg(vRegD dst, vRegF src) %{
14979   match(Set dst (ConvF2D src));
14980 
14981   ins_cost(INSN_COST * 5);
14982   format %{ "fcvts  $dst, $src \t// f2d" %}
14983 
14984   ins_encode %{
14985     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14986   %}
14987 
14988   ins_pipe(fp_f2d);
14989 %}
14990 
14991 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14992   match(Set dst (ConvF2I src));
14993 
14994   ins_cost(INSN_COST * 5);
14995   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14996 
14997   ins_encode %{
14998     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14999   %}
15000 
15001   ins_pipe(fp_f2i);
15002 %}
15003 
15004 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15005   match(Set dst (ConvF2L src));
15006 
15007   ins_cost(INSN_COST * 5);
15008   format %{ "fcvtzs  $dst, $src \t// f2l" %}
15009 
15010   ins_encode %{
15011     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15012   %}
15013 
15014   ins_pipe(fp_f2l);
15015 %}
15016 
15017 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15018   match(Set dst (ConvF2HF src));
15019   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15020             "smov $dst, $tmp\t# move result from $tmp to $dst"
15021   %}
15022   effect(TEMP tmp);
15023   ins_encode %{
15024       __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
15025       __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
15026   %}
15027   ins_pipe(pipe_slow);
15028 %}
15029 
15030 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15031   match(Set dst (ConvHF2F src));
15032   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15033             "fcvt $dst, $tmp\t# convert half to single precision"
15034   %}
15035   effect(TEMP tmp);
15036   ins_encode %{
15037       __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
15038       __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
15039   %}
15040   ins_pipe(pipe_slow);
15041 %}
15042 
15043 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15044   match(Set dst (ConvI2F src));
15045 
15046   ins_cost(INSN_COST * 5);
15047   format %{ "scvtfws  $dst, $src \t// i2f" %}
15048 
15049   ins_encode %{
15050     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15051   %}
15052 
15053   ins_pipe(fp_i2f);
15054 %}
15055 
15056 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15057   match(Set dst (ConvL2F src));
15058 
15059   ins_cost(INSN_COST * 5);
15060   format %{ "scvtfs  $dst, $src \t// l2f" %}
15061 
15062   ins_encode %{
15063     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15064   %}
15065 
15066   ins_pipe(fp_l2f);
15067 %}
15068 
15069 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15070   match(Set dst (ConvD2I src));
15071 
15072   ins_cost(INSN_COST * 5);
15073   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15074 
15075   ins_encode %{
15076     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15077   %}
15078 
15079   ins_pipe(fp_d2i);
15080 %}
15081 
15082 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15083   match(Set dst (ConvD2L src));
15084 
15085   ins_cost(INSN_COST * 5);
15086   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15087 
15088   ins_encode %{
15089     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15090   %}
15091 
15092   ins_pipe(fp_d2l);
15093 %}
15094 
15095 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15096   match(Set dst (ConvI2D src));
15097 
15098   ins_cost(INSN_COST * 5);
15099   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15100 
15101   ins_encode %{
15102     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15103   %}
15104 
15105   ins_pipe(fp_i2d);
15106 %}
15107 
15108 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15109   match(Set dst (ConvL2D src));
15110 
15111   ins_cost(INSN_COST * 5);
15112   format %{ "scvtfd  $dst, $src \t// l2d" %}
15113 
15114   ins_encode %{
15115     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15116   %}
15117 
15118   ins_pipe(fp_l2d);
15119 %}
15120 
15121 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15122 %{
15123   match(Set dst (RoundD src));
15124   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15125   format %{ "java_round_double $dst,$src"%}
15126   ins_encode %{
15127     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15128                          as_FloatRegister($ftmp$$reg));
15129   %}
15130   ins_pipe(pipe_slow);
15131 %}
15132 
15133 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15134 %{
15135   match(Set dst (RoundF src));
15136   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15137   format %{ "java_round_float $dst,$src"%}
15138   ins_encode %{
15139     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15140                         as_FloatRegister($ftmp$$reg));
15141   %}
15142   ins_pipe(pipe_slow);
15143 %}
15144 
15145 // stack <-> reg and reg <-> reg shuffles with no conversion
15146 
15147 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15148 
15149   match(Set dst (MoveF2I src));
15150 
15151   effect(DEF dst, USE src);
15152 
15153   ins_cost(4 * INSN_COST);
15154 
15155   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15156 
15157   ins_encode %{
15158     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15159   %}
15160 
15161   ins_pipe(iload_reg_reg);
15162 
15163 %}
15164 
15165 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15166 
15167   match(Set dst (MoveI2F src));
15168 
15169   effect(DEF dst, USE src);
15170 
15171   ins_cost(4 * INSN_COST);
15172 
15173   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15174 
15175   ins_encode %{
15176     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15177   %}
15178 
15179   ins_pipe(pipe_class_memory);
15180 
15181 %}
15182 
15183 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15184 
15185   match(Set dst (MoveD2L src));
15186 
15187   effect(DEF dst, USE src);
15188 
15189   ins_cost(4 * INSN_COST);
15190 
15191   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15192 
15193   ins_encode %{
15194     __ ldr($dst$$Register, Address(sp, $src$$disp));
15195   %}
15196 
15197   ins_pipe(iload_reg_reg);
15198 
15199 %}
15200 
15201 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15202 
15203   match(Set dst (MoveL2D src));
15204 
15205   effect(DEF dst, USE src);
15206 
15207   ins_cost(4 * INSN_COST);
15208 
15209   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15210 
15211   ins_encode %{
15212     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15213   %}
15214 
15215   ins_pipe(pipe_class_memory);
15216 
15217 %}
15218 
15219 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15220 
15221   match(Set dst (MoveF2I src));
15222 
15223   effect(DEF dst, USE src);
15224 
15225   ins_cost(INSN_COST);
15226 
15227   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15228 
15229   ins_encode %{
15230     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15231   %}
15232 
15233   ins_pipe(pipe_class_memory);
15234 
15235 %}
15236 
15237 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15238 
15239   match(Set dst (MoveI2F src));
15240 
15241   effect(DEF dst, USE src);
15242 
15243   ins_cost(INSN_COST);
15244 
15245   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15246 
15247   ins_encode %{
15248     __ strw($src$$Register, Address(sp, $dst$$disp));
15249   %}
15250 
15251   ins_pipe(istore_reg_reg);
15252 
15253 %}
15254 
15255 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15256 
15257   match(Set dst (MoveD2L src));
15258 
15259   effect(DEF dst, USE src);
15260 
15261   ins_cost(INSN_COST);
15262 
15263   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15264 
15265   ins_encode %{
15266     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15267   %}
15268 
15269   ins_pipe(pipe_class_memory);
15270 
15271 %}
15272 
15273 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15274 
15275   match(Set dst (MoveL2D src));
15276 
15277   effect(DEF dst, USE src);
15278 
15279   ins_cost(INSN_COST);
15280 
15281   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15282 
15283   ins_encode %{
15284     __ str($src$$Register, Address(sp, $dst$$disp));
15285   %}
15286 
15287   ins_pipe(istore_reg_reg);
15288 
15289 %}
15290 
15291 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15292 
15293   match(Set dst (MoveF2I src));
15294 
15295   effect(DEF dst, USE src);
15296 
15297   ins_cost(INSN_COST);
15298 
15299   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15300 
15301   ins_encode %{
15302     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15303   %}
15304 
15305   ins_pipe(fp_f2i);
15306 
15307 %}
15308 
15309 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15310 
15311   match(Set dst (MoveI2F src));
15312 
15313   effect(DEF dst, USE src);
15314 
15315   ins_cost(INSN_COST);
15316 
15317   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15318 
15319   ins_encode %{
15320     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15321   %}
15322 
15323   ins_pipe(fp_i2f);
15324 
15325 %}
15326 
15327 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15328 
15329   match(Set dst (MoveD2L src));
15330 
15331   effect(DEF dst, USE src);
15332 
15333   ins_cost(INSN_COST);
15334 
15335   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15336 
15337   ins_encode %{
15338     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15339   %}
15340 
15341   ins_pipe(fp_d2l);
15342 
15343 %}
15344 
15345 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15346 
15347   match(Set dst (MoveL2D src));
15348 
15349   effect(DEF dst, USE src);
15350 
15351   ins_cost(INSN_COST);
15352 
15353   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15354 
15355   ins_encode %{
15356     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15357   %}
15358 
15359   ins_pipe(fp_l2d);
15360 
15361 %}
15362 
15363 // ============================================================================
15364 // clearing of an array
15365 
15366 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15367 %{
15368   match(Set dummy (ClearArray cnt base));
15369   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15370 
15371   ins_cost(4 * INSN_COST);
15372   format %{ "ClearArray $cnt, $base" %}
15373 
15374   ins_encode %{
15375     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15376     if (tpc == NULL) {
15377       ciEnv::current()->record_failure("CodeCache is full");
15378       return;
15379     }
15380   %}
15381 
15382   ins_pipe(pipe_class_memory);
15383 %}
15384 
15385 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15386 %{
15387   predicate((uint64_t)n->in(2)->get_long()
15388             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15389   match(Set dummy (ClearArray cnt base));
15390   effect(TEMP temp, USE_KILL base, KILL cr);
15391 
15392   ins_cost(4 * INSN_COST);
15393   format %{ "ClearArray $cnt, $base" %}
15394 
15395   ins_encode %{
15396     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15397     if (tpc == NULL) {
15398       ciEnv::current()->record_failure("CodeCache is full");
15399       return;
15400     }
15401   %}
15402 
15403   ins_pipe(pipe_class_memory);
15404 %}
15405 
15406 // ============================================================================
15407 // Overflow Math Instructions
15408 
15409 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15410 %{
15411   match(Set cr (OverflowAddI op1 op2));
15412 
15413   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15414   ins_cost(INSN_COST);
15415   ins_encode %{
15416     __ cmnw($op1$$Register, $op2$$Register);
15417   %}
15418 
15419   ins_pipe(icmp_reg_reg);
15420 %}
15421 
15422 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15423 %{
15424   match(Set cr (OverflowAddI op1 op2));
15425 
15426   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15427   ins_cost(INSN_COST);
15428   ins_encode %{
15429     __ cmnw($op1$$Register, $op2$$constant);
15430   %}
15431 
15432   ins_pipe(icmp_reg_imm);
15433 %}
15434 
15435 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15436 %{
15437   match(Set cr (OverflowAddL op1 op2));
15438 
15439   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15440   ins_cost(INSN_COST);
15441   ins_encode %{
15442     __ cmn($op1$$Register, $op2$$Register);
15443   %}
15444 
15445   ins_pipe(icmp_reg_reg);
15446 %}
15447 
15448 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15449 %{
15450   match(Set cr (OverflowAddL op1 op2));
15451 
15452   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15453   ins_cost(INSN_COST);
15454   ins_encode %{
15455     __ adds(zr, $op1$$Register, $op2$$constant);
15456   %}
15457 
15458   ins_pipe(icmp_reg_imm);
15459 %}
15460 
15461 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15462 %{
15463   match(Set cr (OverflowSubI op1 op2));
15464 
15465   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15466   ins_cost(INSN_COST);
15467   ins_encode %{
15468     __ cmpw($op1$$Register, $op2$$Register);
15469   %}
15470 
15471   ins_pipe(icmp_reg_reg);
15472 %}
15473 
15474 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15475 %{
15476   match(Set cr (OverflowSubI op1 op2));
15477 
15478   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15479   ins_cost(INSN_COST);
15480   ins_encode %{
15481     __ cmpw($op1$$Register, $op2$$constant);
15482   %}
15483 
15484   ins_pipe(icmp_reg_imm);
15485 %}
15486 
15487 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15488 %{
15489   match(Set cr (OverflowSubL op1 op2));
15490 
15491   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15492   ins_cost(INSN_COST);
15493   ins_encode %{
15494     __ cmp($op1$$Register, $op2$$Register);
15495   %}
15496 
15497   ins_pipe(icmp_reg_reg);
15498 %}
15499 
15500 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15501 %{
15502   match(Set cr (OverflowSubL op1 op2));
15503 
15504   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15505   ins_cost(INSN_COST);
15506   ins_encode %{
15507     __ subs(zr, $op1$$Register, $op2$$constant);
15508   %}
15509 
15510   ins_pipe(icmp_reg_imm);
15511 %}
15512 
15513 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15514 %{
15515   match(Set cr (OverflowSubI zero op1));
15516 
15517   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15518   ins_cost(INSN_COST);
15519   ins_encode %{
15520     __ cmpw(zr, $op1$$Register);
15521   %}
15522 
15523   ins_pipe(icmp_reg_imm);
15524 %}
15525 
15526 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15527 %{
15528   match(Set cr (OverflowSubL zero op1));
15529 
15530   format %{ "cmp   zr, $op1\t# overflow check long" %}
15531   ins_cost(INSN_COST);
15532   ins_encode %{
15533     __ cmp(zr, $op1$$Register);
15534   %}
15535 
15536   ins_pipe(icmp_reg_imm);
15537 %}
15538 
15539 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15540 %{
15541   match(Set cr (OverflowMulI op1 op2));
15542 
15543   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15544             "cmp   rscratch1, rscratch1, sxtw\n\t"
15545             "movw  rscratch1, #0x80000000\n\t"
15546             "cselw rscratch1, rscratch1, zr, NE\n\t"
15547             "cmpw  rscratch1, #1" %}
15548   ins_cost(5 * INSN_COST);
15549   ins_encode %{
15550     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15551     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15552     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15553     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15554     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15555   %}
15556 
15557   ins_pipe(pipe_slow);
15558 %}
15559 
15560 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15561 %{
15562   match(If cmp (OverflowMulI op1 op2));
15563   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15564             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15565   effect(USE labl, KILL cr);
15566 
15567   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15568             "cmp   rscratch1, rscratch1, sxtw\n\t"
15569             "b$cmp   $labl" %}
15570   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15571   ins_encode %{
15572     Label* L = $labl$$label;
15573     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15574     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15575     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15576     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15577   %}
15578 
15579   ins_pipe(pipe_serial);
15580 %}
15581 
15582 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15583 %{
15584   match(Set cr (OverflowMulL op1 op2));
15585 
15586   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15587             "smulh rscratch2, $op1, $op2\n\t"
15588             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15589             "movw  rscratch1, #0x80000000\n\t"
15590             "cselw rscratch1, rscratch1, zr, NE\n\t"
15591             "cmpw  rscratch1, #1" %}
15592   ins_cost(6 * INSN_COST);
15593   ins_encode %{
15594     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15595     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15596     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15597     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15598     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15599     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15600   %}
15601 
15602   ins_pipe(pipe_slow);
15603 %}
15604 
15605 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15606 %{
15607   match(If cmp (OverflowMulL op1 op2));
15608   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15609             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15610   effect(USE labl, KILL cr);
15611 
15612   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15613             "smulh rscratch2, $op1, $op2\n\t"
15614             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15615             "b$cmp $labl" %}
15616   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15617   ins_encode %{
15618     Label* L = $labl$$label;
15619     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15620     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15621     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15622     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15623     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15624   %}
15625 
15626   ins_pipe(pipe_serial);
15627 %}
15628 
15629 // ============================================================================
15630 // Compare Instructions
15631 
15632 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15633 %{
15634   match(Set cr (CmpI op1 op2));
15635 
15636   effect(DEF cr, USE op1, USE op2);
15637 
15638   ins_cost(INSN_COST);
15639   format %{ "cmpw  $op1, $op2" %}
15640 
15641   ins_encode(aarch64_enc_cmpw(op1, op2));
15642 
15643   ins_pipe(icmp_reg_reg);
15644 %}
15645 
15646 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15647 %{
15648   match(Set cr (CmpI op1 zero));
15649 
15650   effect(DEF cr, USE op1);
15651 
15652   ins_cost(INSN_COST);
15653   format %{ "cmpw $op1, 0" %}
15654 
15655   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15656 
15657   ins_pipe(icmp_reg_imm);
15658 %}
15659 
15660 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15661 %{
15662   match(Set cr (CmpI op1 op2));
15663 
15664   effect(DEF cr, USE op1);
15665 
15666   ins_cost(INSN_COST);
15667   format %{ "cmpw  $op1, $op2" %}
15668 
15669   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15670 
15671   ins_pipe(icmp_reg_imm);
15672 %}
15673 
15674 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15675 %{
15676   match(Set cr (CmpI op1 op2));
15677 
15678   effect(DEF cr, USE op1);
15679 
15680   ins_cost(INSN_COST * 2);
15681   format %{ "cmpw  $op1, $op2" %}
15682 
15683   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15684 
15685   ins_pipe(icmp_reg_imm);
15686 %}
15687 
15688 // Unsigned compare Instructions; really, same as signed compare
15689 // except it should only be used to feed an If or a CMovI which takes a
15690 // cmpOpU.
15691 
15692 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15693 %{
15694   match(Set cr (CmpU op1 op2));
15695 
15696   effect(DEF cr, USE op1, USE op2);
15697 
15698   ins_cost(INSN_COST);
15699   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15700 
15701   ins_encode(aarch64_enc_cmpw(op1, op2));
15702 
15703   ins_pipe(icmp_reg_reg);
15704 %}
15705 
15706 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15707 %{
15708   match(Set cr (CmpU op1 zero));
15709 
15710   effect(DEF cr, USE op1);
15711 
15712   ins_cost(INSN_COST);
15713   format %{ "cmpw $op1, #0\t# unsigned" %}
15714 
15715   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15716 
15717   ins_pipe(icmp_reg_imm);
15718 %}
15719 
15720 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15721 %{
15722   match(Set cr (CmpU op1 op2));
15723 
15724   effect(DEF cr, USE op1);
15725 
15726   ins_cost(INSN_COST);
15727   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15728 
15729   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15730 
15731   ins_pipe(icmp_reg_imm);
15732 %}
15733 
15734 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15735 %{
15736   match(Set cr (CmpU op1 op2));
15737 
15738   effect(DEF cr, USE op1);
15739 
15740   ins_cost(INSN_COST * 2);
15741   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15742 
15743   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15744 
15745   ins_pipe(icmp_reg_imm);
15746 %}
15747 
15748 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15749 %{
15750   match(Set cr (CmpL op1 op2));
15751 
15752   effect(DEF cr, USE op1, USE op2);
15753 
15754   ins_cost(INSN_COST);
15755   format %{ "cmp  $op1, $op2" %}
15756 
15757   ins_encode(aarch64_enc_cmp(op1, op2));
15758 
15759   ins_pipe(icmp_reg_reg);
15760 %}
15761 
15762 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15763 %{
15764   match(Set cr (CmpL op1 zero));
15765 
15766   effect(DEF cr, USE op1);
15767 
15768   ins_cost(INSN_COST);
15769   format %{ "tst  $op1" %}
15770 
15771   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15772 
15773   ins_pipe(icmp_reg_imm);
15774 %}
15775 
15776 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15777 %{
15778   match(Set cr (CmpL op1 op2));
15779 
15780   effect(DEF cr, USE op1);
15781 
15782   ins_cost(INSN_COST);
15783   format %{ "cmp  $op1, $op2" %}
15784 
15785   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15786 
15787   ins_pipe(icmp_reg_imm);
15788 %}
15789 
15790 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15791 %{
15792   match(Set cr (CmpL op1 op2));
15793 
15794   effect(DEF cr, USE op1);
15795 
15796   ins_cost(INSN_COST * 2);
15797   format %{ "cmp  $op1, $op2" %}
15798 
15799   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15800 
15801   ins_pipe(icmp_reg_imm);
15802 %}
15803 
15804 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15805 %{
15806   match(Set cr (CmpUL op1 op2));
15807 
15808   effect(DEF cr, USE op1, USE op2);
15809 
15810   ins_cost(INSN_COST);
15811   format %{ "cmp  $op1, $op2" %}
15812 
15813   ins_encode(aarch64_enc_cmp(op1, op2));
15814 
15815   ins_pipe(icmp_reg_reg);
15816 %}
15817 
15818 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15819 %{
15820   match(Set cr (CmpUL op1 zero));
15821 
15822   effect(DEF cr, USE op1);
15823 
15824   ins_cost(INSN_COST);
15825   format %{ "tst  $op1" %}
15826 
15827   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15828 
15829   ins_pipe(icmp_reg_imm);
15830 %}
15831 
15832 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15833 %{
15834   match(Set cr (CmpUL op1 op2));
15835 
15836   effect(DEF cr, USE op1);
15837 
15838   ins_cost(INSN_COST);
15839   format %{ "cmp  $op1, $op2" %}
15840 
15841   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15842 
15843   ins_pipe(icmp_reg_imm);
15844 %}
15845 
15846 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15847 %{
15848   match(Set cr (CmpUL op1 op2));
15849 
15850   effect(DEF cr, USE op1);
15851 
15852   ins_cost(INSN_COST * 2);
15853   format %{ "cmp  $op1, $op2" %}
15854 
15855   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15856 
15857   ins_pipe(icmp_reg_imm);
15858 %}
15859 
15860 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15861 %{
15862   match(Set cr (CmpP op1 op2));
15863 
15864   effect(DEF cr, USE op1, USE op2);
15865 
15866   ins_cost(INSN_COST);
15867   format %{ "cmp  $op1, $op2\t // ptr" %}
15868 
15869   ins_encode(aarch64_enc_cmpp(op1, op2));
15870 
15871   ins_pipe(icmp_reg_reg);
15872 %}
15873 
15874 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15875 %{
15876   match(Set cr (CmpN op1 op2));
15877 
15878   effect(DEF cr, USE op1, USE op2);
15879 
15880   ins_cost(INSN_COST);
15881   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15882 
15883   ins_encode(aarch64_enc_cmpn(op1, op2));
15884 
15885   ins_pipe(icmp_reg_reg);
15886 %}
15887 
15888 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15889 %{
15890   match(Set cr (CmpP op1 zero));
15891 
15892   effect(DEF cr, USE op1, USE zero);
15893 
15894   ins_cost(INSN_COST);
15895   format %{ "cmp  $op1, 0\t // ptr" %}
15896 
15897   ins_encode(aarch64_enc_testp(op1));
15898 
15899   ins_pipe(icmp_reg_imm);
15900 %}
15901 
15902 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15903 %{
15904   match(Set cr (CmpN op1 zero));
15905 
15906   effect(DEF cr, USE op1, USE zero);
15907 
15908   ins_cost(INSN_COST);
15909   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15910 
15911   ins_encode(aarch64_enc_testn(op1));
15912 
15913   ins_pipe(icmp_reg_imm);
15914 %}
15915 
15916 // FP comparisons
15917 //
15918 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15919 // using normal cmpOp. See declaration of rFlagsReg for details.
15920 
15921 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15922 %{
15923   match(Set cr (CmpF src1 src2));
15924 
15925   ins_cost(3 * INSN_COST);
15926   format %{ "fcmps $src1, $src2" %}
15927 
15928   ins_encode %{
15929     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15930   %}
15931 
15932   ins_pipe(pipe_class_compare);
15933 %}
15934 
15935 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15936 %{
15937   match(Set cr (CmpF src1 src2));
15938 
15939   ins_cost(3 * INSN_COST);
15940   format %{ "fcmps $src1, 0.0" %}
15941 
15942   ins_encode %{
15943     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15944   %}
15945 
15946   ins_pipe(pipe_class_compare);
15947 %}
15948 // FROM HERE
15949 
15950 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15951 %{
15952   match(Set cr (CmpD src1 src2));
15953 
15954   ins_cost(3 * INSN_COST);
15955   format %{ "fcmpd $src1, $src2" %}
15956 
15957   ins_encode %{
15958     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15959   %}
15960 
15961   ins_pipe(pipe_class_compare);
15962 %}
15963 
15964 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15965 %{
15966   match(Set cr (CmpD src1 src2));
15967 
15968   ins_cost(3 * INSN_COST);
15969   format %{ "fcmpd $src1, 0.0" %}
15970 
15971   ins_encode %{
15972     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15973   %}
15974 
15975   ins_pipe(pipe_class_compare);
15976 %}
15977 
15978 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15979 %{
15980   match(Set dst (CmpF3 src1 src2));
15981   effect(KILL cr);
15982 
15983   ins_cost(5 * INSN_COST);
15984   format %{ "fcmps $src1, $src2\n\t"
15985             "csinvw($dst, zr, zr, eq\n\t"
15986             "csnegw($dst, $dst, $dst, lt)"
15987   %}
15988 
15989   ins_encode %{
15990     Label done;
15991     FloatRegister s1 = as_FloatRegister($src1$$reg);
15992     FloatRegister s2 = as_FloatRegister($src2$$reg);
15993     Register d = as_Register($dst$$reg);
15994     __ fcmps(s1, s2);
15995     // installs 0 if EQ else -1
15996     __ csinvw(d, zr, zr, Assembler::EQ);
15997     // keeps -1 if less or unordered else installs 1
15998     __ csnegw(d, d, d, Assembler::LT);
15999     __ bind(done);
16000   %}
16001 
16002   ins_pipe(pipe_class_default);
16003 
16004 %}
16005 
16006 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16007 %{
16008   match(Set dst (CmpD3 src1 src2));
16009   effect(KILL cr);
16010 
16011   ins_cost(5 * INSN_COST);
16012   format %{ "fcmpd $src1, $src2\n\t"
16013             "csinvw($dst, zr, zr, eq\n\t"
16014             "csnegw($dst, $dst, $dst, lt)"
16015   %}
16016 
16017   ins_encode %{
16018     Label done;
16019     FloatRegister s1 = as_FloatRegister($src1$$reg);
16020     FloatRegister s2 = as_FloatRegister($src2$$reg);
16021     Register d = as_Register($dst$$reg);
16022     __ fcmpd(s1, s2);
16023     // installs 0 if EQ else -1
16024     __ csinvw(d, zr, zr, Assembler::EQ);
16025     // keeps -1 if less or unordered else installs 1
16026     __ csnegw(d, d, d, Assembler::LT);
16027     __ bind(done);
16028   %}
16029   ins_pipe(pipe_class_default);
16030 
16031 %}
16032 
16033 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16034 %{
16035   match(Set dst (CmpF3 src1 zero));
16036   effect(KILL cr);
16037 
16038   ins_cost(5 * INSN_COST);
16039   format %{ "fcmps $src1, 0.0\n\t"
16040             "csinvw($dst, zr, zr, eq\n\t"
16041             "csnegw($dst, $dst, $dst, lt)"
16042   %}
16043 
16044   ins_encode %{
16045     Label done;
16046     FloatRegister s1 = as_FloatRegister($src1$$reg);
16047     Register d = as_Register($dst$$reg);
16048     __ fcmps(s1, 0.0);
16049     // installs 0 if EQ else -1
16050     __ csinvw(d, zr, zr, Assembler::EQ);
16051     // keeps -1 if less or unordered else installs 1
16052     __ csnegw(d, d, d, Assembler::LT);
16053     __ bind(done);
16054   %}
16055 
16056   ins_pipe(pipe_class_default);
16057 
16058 %}
16059 
16060 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16061 %{
16062   match(Set dst (CmpD3 src1 zero));
16063   effect(KILL cr);
16064 
16065   ins_cost(5 * INSN_COST);
16066   format %{ "fcmpd $src1, 0.0\n\t"
16067             "csinvw($dst, zr, zr, eq\n\t"
16068             "csnegw($dst, $dst, $dst, lt)"
16069   %}
16070 
16071   ins_encode %{
16072     Label done;
16073     FloatRegister s1 = as_FloatRegister($src1$$reg);
16074     Register d = as_Register($dst$$reg);
16075     __ fcmpd(s1, 0.0);
16076     // installs 0 if EQ else -1
16077     __ csinvw(d, zr, zr, Assembler::EQ);
16078     // keeps -1 if less or unordered else installs 1
16079     __ csnegw(d, d, d, Assembler::LT);
16080     __ bind(done);
16081   %}
16082   ins_pipe(pipe_class_default);
16083 
16084 %}
16085 
16086 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16087 %{
16088   match(Set dst (CmpLTMask p q));
16089   effect(KILL cr);
16090 
16091   ins_cost(3 * INSN_COST);
16092 
16093   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16094             "csetw $dst, lt\n\t"
16095             "subw $dst, zr, $dst"
16096   %}
16097 
16098   ins_encode %{
16099     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16100     __ csetw(as_Register($dst$$reg), Assembler::LT);
16101     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16102   %}
16103 
16104   ins_pipe(ialu_reg_reg);
16105 %}
16106 
16107 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16108 %{
16109   match(Set dst (CmpLTMask src zero));
16110   effect(KILL cr);
16111 
16112   ins_cost(INSN_COST);
16113 
16114   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16115 
16116   ins_encode %{
16117     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16118   %}
16119 
16120   ins_pipe(ialu_reg_shift);
16121 %}
16122 
16123 // ============================================================================
16124 // Max and Min
16125 
16126 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16127 
16128 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16129 %{
16130   effect(DEF cr, USE src);
16131   ins_cost(INSN_COST);
16132   format %{ "cmpw $src, 0" %}
16133 
16134   ins_encode %{
16135     __ cmpw($src$$Register, 0);
16136   %}
16137   ins_pipe(icmp_reg_imm);
16138 %}
16139 
16140 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16141 %{
16142   match(Set dst (MinI src1 src2));
16143   ins_cost(INSN_COST * 3);
16144 
16145   expand %{
16146     rFlagsReg cr;
16147     compI_reg_reg(cr, src1, src2);
16148     cmovI_reg_reg_lt(dst, src1, src2, cr);
16149   %}
16150 %}
16151 
16152 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16153 %{
16154   match(Set dst (MaxI src1 src2));
16155   ins_cost(INSN_COST * 3);
16156 
16157   expand %{
16158     rFlagsReg cr;
16159     compI_reg_reg(cr, src1, src2);
16160     cmovI_reg_reg_gt(dst, src1, src2, cr);
16161   %}
16162 %}
16163 
16164 
16165 // ============================================================================
16166 // Branch Instructions
16167 
16168 // Direct Branch.
16169 instruct branch(label lbl)
16170 %{
16171   match(Goto);
16172 
16173   effect(USE lbl);
16174 
16175   ins_cost(BRANCH_COST);
16176   format %{ "b  $lbl" %}
16177 
16178   ins_encode(aarch64_enc_b(lbl));
16179 
16180   ins_pipe(pipe_branch);
16181 %}
16182 
16183 // Conditional Near Branch
16184 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16185 %{
16186   // Same match rule as `branchConFar'.
16187   match(If cmp cr);
16188 
16189   effect(USE lbl);
16190 
16191   ins_cost(BRANCH_COST);
16192   // If set to 1 this indicates that the current instruction is a
16193   // short variant of a long branch. This avoids using this
16194   // instruction in first-pass matching. It will then only be used in
16195   // the `Shorten_branches' pass.
16196   // ins_short_branch(1);
16197   format %{ "b$cmp  $lbl" %}
16198 
16199   ins_encode(aarch64_enc_br_con(cmp, lbl));
16200 
16201   ins_pipe(pipe_branch_cond);
16202 %}
16203 
16204 // Conditional Near Branch Unsigned
16205 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16206 %{
16207   // Same match rule as `branchConFar'.
16208   match(If cmp cr);
16209 
16210   effect(USE lbl);
16211 
16212   ins_cost(BRANCH_COST);
16213   // If set to 1 this indicates that the current instruction is a
16214   // short variant of a long branch. This avoids using this
16215   // instruction in first-pass matching. It will then only be used in
16216   // the `Shorten_branches' pass.
16217   // ins_short_branch(1);
16218   format %{ "b$cmp  $lbl\t# unsigned" %}
16219 
16220   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16221 
16222   ins_pipe(pipe_branch_cond);
16223 %}
16224 
16225 // Make use of CBZ and CBNZ.  These instructions, as well as being
16226 // shorter than (cmp; branch), have the additional benefit of not
16227 // killing the flags.
16228 
16229 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16230   match(If cmp (CmpI op1 op2));
16231   effect(USE labl);
16232 
16233   ins_cost(BRANCH_COST);
16234   format %{ "cbw$cmp   $op1, $labl" %}
16235   ins_encode %{
16236     Label* L = $labl$$label;
16237     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16238     if (cond == Assembler::EQ)
16239       __ cbzw($op1$$Register, *L);
16240     else
16241       __ cbnzw($op1$$Register, *L);
16242   %}
16243   ins_pipe(pipe_cmp_branch);
16244 %}
16245 
16246 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16247   match(If cmp (CmpL op1 op2));
16248   effect(USE labl);
16249 
16250   ins_cost(BRANCH_COST);
16251   format %{ "cb$cmp   $op1, $labl" %}
16252   ins_encode %{
16253     Label* L = $labl$$label;
16254     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16255     if (cond == Assembler::EQ)
16256       __ cbz($op1$$Register, *L);
16257     else
16258       __ cbnz($op1$$Register, *L);
16259   %}
16260   ins_pipe(pipe_cmp_branch);
16261 %}
16262 
16263 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16264   match(If cmp (CmpP op1 op2));
16265   effect(USE labl);
16266 
16267   ins_cost(BRANCH_COST);
16268   format %{ "cb$cmp   $op1, $labl" %}
16269   ins_encode %{
16270     Label* L = $labl$$label;
16271     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16272     if (cond == Assembler::EQ)
16273       __ cbz($op1$$Register, *L);
16274     else
16275       __ cbnz($op1$$Register, *L);
16276   %}
16277   ins_pipe(pipe_cmp_branch);
16278 %}
16279 
16280 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16281   match(If cmp (CmpN op1 op2));
16282   effect(USE labl);
16283 
16284   ins_cost(BRANCH_COST);
16285   format %{ "cbw$cmp   $op1, $labl" %}
16286   ins_encode %{
16287     Label* L = $labl$$label;
16288     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16289     if (cond == Assembler::EQ)
16290       __ cbzw($op1$$Register, *L);
16291     else
16292       __ cbnzw($op1$$Register, *L);
16293   %}
16294   ins_pipe(pipe_cmp_branch);
16295 %}
16296 
16297 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16298   match(If cmp (CmpP (DecodeN oop) zero));
16299   effect(USE labl);
16300 
16301   ins_cost(BRANCH_COST);
16302   format %{ "cb$cmp   $oop, $labl" %}
16303   ins_encode %{
16304     Label* L = $labl$$label;
16305     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16306     if (cond == Assembler::EQ)
16307       __ cbzw($oop$$Register, *L);
16308     else
16309       __ cbnzw($oop$$Register, *L);
16310   %}
16311   ins_pipe(pipe_cmp_branch);
16312 %}
16313 
16314 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16315   match(If cmp (CmpU op1 op2));
16316   effect(USE labl);
16317 
16318   ins_cost(BRANCH_COST);
16319   format %{ "cbw$cmp   $op1, $labl" %}
16320   ins_encode %{
16321     Label* L = $labl$$label;
16322     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16323     if (cond == Assembler::EQ || cond == Assembler::LS)
16324       __ cbzw($op1$$Register, *L);
16325     else
16326       __ cbnzw($op1$$Register, *L);
16327   %}
16328   ins_pipe(pipe_cmp_branch);
16329 %}
16330 
16331 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16332   match(If cmp (CmpUL op1 op2));
16333   effect(USE labl);
16334 
16335   ins_cost(BRANCH_COST);
16336   format %{ "cb$cmp   $op1, $labl" %}
16337   ins_encode %{
16338     Label* L = $labl$$label;
16339     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16340     if (cond == Assembler::EQ || cond == Assembler::LS)
16341       __ cbz($op1$$Register, *L);
16342     else
16343       __ cbnz($op1$$Register, *L);
16344   %}
16345   ins_pipe(pipe_cmp_branch);
16346 %}
16347 
16348 // Test bit and Branch
16349 
16350 // Patterns for short (< 32KiB) variants
16351 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16352   match(If cmp (CmpL op1 op2));
16353   effect(USE labl);
16354 
16355   ins_cost(BRANCH_COST);
16356   format %{ "cb$cmp   $op1, $labl # long" %}
16357   ins_encode %{
16358     Label* L = $labl$$label;
16359     Assembler::Condition cond =
16360       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16361     __ tbr(cond, $op1$$Register, 63, *L);
16362   %}
16363   ins_pipe(pipe_cmp_branch);
16364   ins_short_branch(1);
16365 %}
16366 
16367 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16368   match(If cmp (CmpI op1 op2));
16369   effect(USE labl);
16370 
16371   ins_cost(BRANCH_COST);
16372   format %{ "cb$cmp   $op1, $labl # int" %}
16373   ins_encode %{
16374     Label* L = $labl$$label;
16375     Assembler::Condition cond =
16376       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16377     __ tbr(cond, $op1$$Register, 31, *L);
16378   %}
16379   ins_pipe(pipe_cmp_branch);
16380   ins_short_branch(1);
16381 %}
16382 
16383 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16384   match(If cmp (CmpL (AndL op1 op2) op3));
16385   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16386   effect(USE labl);
16387 
16388   ins_cost(BRANCH_COST);
16389   format %{ "tb$cmp   $op1, $op2, $labl" %}
16390   ins_encode %{
16391     Label* L = $labl$$label;
16392     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16393     int bit = exact_log2_long($op2$$constant);
16394     __ tbr(cond, $op1$$Register, bit, *L);
16395   %}
16396   ins_pipe(pipe_cmp_branch);
16397   ins_short_branch(1);
16398 %}
16399 
16400 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16401   match(If cmp (CmpI (AndI op1 op2) op3));
16402   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16403   effect(USE labl);
16404 
16405   ins_cost(BRANCH_COST);
16406   format %{ "tb$cmp   $op1, $op2, $labl" %}
16407   ins_encode %{
16408     Label* L = $labl$$label;
16409     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16410     int bit = exact_log2((juint)$op2$$constant);
16411     __ tbr(cond, $op1$$Register, bit, *L);
16412   %}
16413   ins_pipe(pipe_cmp_branch);
16414   ins_short_branch(1);
16415 %}
16416 
16417 // And far variants
16418 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16419   match(If cmp (CmpL op1 op2));
16420   effect(USE labl);
16421 
16422   ins_cost(BRANCH_COST);
16423   format %{ "cb$cmp   $op1, $labl # long" %}
16424   ins_encode %{
16425     Label* L = $labl$$label;
16426     Assembler::Condition cond =
16427       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16428     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16429   %}
16430   ins_pipe(pipe_cmp_branch);
16431 %}
16432 
16433 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16434   match(If cmp (CmpI op1 op2));
16435   effect(USE labl);
16436 
16437   ins_cost(BRANCH_COST);
16438   format %{ "cb$cmp   $op1, $labl # int" %}
16439   ins_encode %{
16440     Label* L = $labl$$label;
16441     Assembler::Condition cond =
16442       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16443     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16444   %}
16445   ins_pipe(pipe_cmp_branch);
16446 %}
16447 
16448 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16449   match(If cmp (CmpL (AndL op1 op2) op3));
16450   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16451   effect(USE labl);
16452 
16453   ins_cost(BRANCH_COST);
16454   format %{ "tb$cmp   $op1, $op2, $labl" %}
16455   ins_encode %{
16456     Label* L = $labl$$label;
16457     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16458     int bit = exact_log2_long($op2$$constant);
16459     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16460   %}
16461   ins_pipe(pipe_cmp_branch);
16462 %}
16463 
16464 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16465   match(If cmp (CmpI (AndI op1 op2) op3));
16466   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16467   effect(USE labl);
16468 
16469   ins_cost(BRANCH_COST);
16470   format %{ "tb$cmp   $op1, $op2, $labl" %}
16471   ins_encode %{
16472     Label* L = $labl$$label;
16473     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16474     int bit = exact_log2((juint)$op2$$constant);
16475     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16476   %}
16477   ins_pipe(pipe_cmp_branch);
16478 %}
16479 
16480 // Test bits
16481 
16482 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16483   match(Set cr (CmpL (AndL op1 op2) op3));
16484   predicate(Assembler::operand_valid_for_logical_immediate
16485             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16486 
16487   ins_cost(INSN_COST);
16488   format %{ "tst $op1, $op2 # long" %}
16489   ins_encode %{
16490     __ tst($op1$$Register, $op2$$constant);
16491   %}
16492   ins_pipe(ialu_reg_reg);
16493 %}
16494 
16495 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16496   match(Set cr (CmpI (AndI op1 op2) op3));
16497   predicate(Assembler::operand_valid_for_logical_immediate
16498             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16499 
16500   ins_cost(INSN_COST);
16501   format %{ "tst $op1, $op2 # int" %}
16502   ins_encode %{
16503     __ tstw($op1$$Register, $op2$$constant);
16504   %}
16505   ins_pipe(ialu_reg_reg);
16506 %}
16507 
16508 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16509   match(Set cr (CmpL (AndL op1 op2) op3));
16510 
16511   ins_cost(INSN_COST);
16512   format %{ "tst $op1, $op2 # long" %}
16513   ins_encode %{
16514     __ tst($op1$$Register, $op2$$Register);
16515   %}
16516   ins_pipe(ialu_reg_reg);
16517 %}
16518 
16519 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16520   match(Set cr (CmpI (AndI op1 op2) op3));
16521 
16522   ins_cost(INSN_COST);
16523   format %{ "tstw $op1, $op2 # int" %}
16524   ins_encode %{
16525     __ tstw($op1$$Register, $op2$$Register);
16526   %}
16527   ins_pipe(ialu_reg_reg);
16528 %}
16529 
16530 
16531 // Conditional Far Branch
16532 // Conditional Far Branch Unsigned
16533 // TODO: fixme
16534 
16535 // counted loop end branch near
16536 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16537 %{
16538   match(CountedLoopEnd cmp cr);
16539 
16540   effect(USE lbl);
16541 
16542   ins_cost(BRANCH_COST);
16543   // short variant.
16544   // ins_short_branch(1);
16545   format %{ "b$cmp $lbl \t// counted loop end" %}
16546 
16547   ins_encode(aarch64_enc_br_con(cmp, lbl));
16548 
16549   ins_pipe(pipe_branch);
16550 %}
16551 
16552 // counted loop end branch near Unsigned
16553 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16554 %{
16555   match(CountedLoopEnd cmp cr);
16556 
16557   effect(USE lbl);
16558 
16559   ins_cost(BRANCH_COST);
16560   // short variant.
16561   // ins_short_branch(1);
16562   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16563 
16564   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16565 
16566   ins_pipe(pipe_branch);
16567 %}
16568 
16569 // counted loop end branch far
16570 // counted loop end branch far unsigned
16571 // TODO: fixme
16572 
16573 // ============================================================================
16574 // inlined locking and unlocking
16575 
16576 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16577 %{
16578   match(Set cr (FastLock object box));
16579   effect(TEMP tmp, TEMP tmp2);
16580 
16581   // TODO
16582   // identify correct cost
16583   ins_cost(5 * INSN_COST);
16584   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16585 
16586   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16587 
16588   ins_pipe(pipe_serial);
16589 %}
16590 
16591 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16592 %{
16593   match(Set cr (FastUnlock object box));
16594   effect(TEMP tmp, TEMP tmp2);
16595 
16596   ins_cost(5 * INSN_COST);
16597   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16598 
16599   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16600 
16601   ins_pipe(pipe_serial);
16602 %}
16603 
16604 
16605 // ============================================================================
16606 // Safepoint Instructions
16607 
16608 // TODO
16609 // provide a near and far version of this code
16610 
16611 instruct safePoint(rFlagsReg cr, iRegP poll)
16612 %{
16613   match(SafePoint poll);
16614   effect(KILL cr);
16615 
16616   format %{
16617     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16618   %}
16619   ins_encode %{
16620     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16621   %}
16622   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16623 %}
16624 
16625 
16626 // ============================================================================
16627 // Procedure Call/Return Instructions
16628 
16629 // Call Java Static Instruction
16630 
16631 instruct CallStaticJavaDirect(method meth)
16632 %{
16633   match(CallStaticJava);
16634 
16635   effect(USE meth);
16636 
16637   ins_cost(CALL_COST);
16638 
16639   format %{ "call,static $meth \t// ==> " %}
16640 
16641   ins_encode(aarch64_enc_java_static_call(meth),
16642              aarch64_enc_call_epilog);
16643 
16644   ins_pipe(pipe_class_call);
16645 %}
16646 
16647 // TO HERE
16648 
16649 // Call Java Dynamic Instruction
16650 instruct CallDynamicJavaDirect(method meth)
16651 %{
16652   match(CallDynamicJava);
16653 
16654   effect(USE meth);
16655 
16656   ins_cost(CALL_COST);
16657 
16658   format %{ "CALL,dynamic $meth \t// ==> " %}
16659 
16660   ins_encode(aarch64_enc_java_dynamic_call(meth),
16661              aarch64_enc_call_epilog);
16662 
16663   ins_pipe(pipe_class_call);
16664 %}
16665 
16666 // Call Runtime Instruction
16667 
16668 instruct CallRuntimeDirect(method meth)
16669 %{
16670   match(CallRuntime);
16671 
16672   effect(USE meth);
16673 
16674   ins_cost(CALL_COST);
16675 
16676   format %{ "CALL, runtime $meth" %}
16677 
16678   ins_encode( aarch64_enc_java_to_runtime(meth) );
16679 
16680   ins_pipe(pipe_class_call);
16681 %}
16682 
16683 // Call Runtime Instruction
16684 
16685 instruct CallLeafDirect(method meth)
16686 %{
16687   match(CallLeaf);
16688 
16689   effect(USE meth);
16690 
16691   ins_cost(CALL_COST);
16692 
16693   format %{ "CALL, runtime leaf $meth" %}
16694 
16695   ins_encode( aarch64_enc_java_to_runtime(meth) );
16696 
16697   ins_pipe(pipe_class_call);
16698 %}
16699 
16700 // Call Runtime Instruction
16701 
16702 instruct CallLeafNoFPDirect(method meth)
16703 %{
16704   match(CallLeafNoFP);
16705 
16706   effect(USE meth);
16707 
16708   ins_cost(CALL_COST);
16709 
16710   format %{ "CALL, runtime leaf nofp $meth" %}
16711 
16712   ins_encode( aarch64_enc_java_to_runtime(meth) );
16713 
16714   ins_pipe(pipe_class_call);
16715 %}
16716 
16717 // Tail Call; Jump from runtime stub to Java code.
16718 // Also known as an 'interprocedural jump'.
16719 // Target of jump will eventually return to caller.
16720 // TailJump below removes the return address.
16721 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16722 %{
16723   match(TailCall jump_target method_ptr);
16724 
16725   ins_cost(CALL_COST);
16726 
16727   format %{ "br $jump_target\t# $method_ptr holds method" %}
16728 
16729   ins_encode(aarch64_enc_tail_call(jump_target));
16730 
16731   ins_pipe(pipe_class_call);
16732 %}
16733 
16734 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16735 %{
16736   match(TailJump jump_target ex_oop);
16737 
16738   ins_cost(CALL_COST);
16739 
16740   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16741 
16742   ins_encode(aarch64_enc_tail_jmp(jump_target));
16743 
16744   ins_pipe(pipe_class_call);
16745 %}
16746 
16747 // Create exception oop: created by stack-crawling runtime code.
16748 // Created exception is now available to this handler, and is setup
16749 // just prior to jumping to this handler. No code emitted.
16750 // TODO check
16751 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16752 instruct CreateException(iRegP_R0 ex_oop)
16753 %{
16754   match(Set ex_oop (CreateEx));
16755 
16756   format %{ " -- \t// exception oop; no code emitted" %}
16757 
16758   size(0);
16759 
16760   ins_encode( /*empty*/ );
16761 
16762   ins_pipe(pipe_class_empty);
16763 %}
16764 
16765 // Rethrow exception: The exception oop will come in the first
16766 // argument position. Then JUMP (not call) to the rethrow stub code.
16767 instruct RethrowException() %{
16768   match(Rethrow);
16769   ins_cost(CALL_COST);
16770 
16771   format %{ "b rethrow_stub" %}
16772 
16773   ins_encode( aarch64_enc_rethrow() );
16774 
16775   ins_pipe(pipe_class_call);
16776 %}
16777 
16778 
16779 // Return Instruction
16780 // epilog node loads ret address into lr as part of frame pop
16781 instruct Ret()
16782 %{
16783   match(Return);
16784 
16785   format %{ "ret\t// return register" %}
16786 
16787   ins_encode( aarch64_enc_ret() );
16788 
16789   ins_pipe(pipe_branch);
16790 %}
16791 
16792 // Die now.
16793 instruct ShouldNotReachHere() %{
16794   match(Halt);
16795 
16796   ins_cost(CALL_COST);
16797   format %{ "ShouldNotReachHere" %}
16798 
16799   ins_encode %{
16800     if (is_reachable()) {
16801       __ stop(_halt_reason);
16802     }
16803   %}
16804 
16805   ins_pipe(pipe_class_default);
16806 %}
16807 
16808 // ============================================================================
16809 // Partial Subtype Check
16810 //
16811 // superklass array for an instance of the superklass.  Set a hidden
16812 // internal cache on a hit (cache is checked with exposed code in
16813 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16814 // encoding ALSO sets flags.
16815 
16816 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16817 %{
16818   match(Set result (PartialSubtypeCheck sub super));
16819   effect(KILL cr, KILL temp);
16820 
16821   ins_cost(1100);  // slightly larger than the next version
16822   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16823 
16824   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16825 
16826   opcode(0x1); // Force zero of result reg on hit
16827 
16828   ins_pipe(pipe_class_memory);
16829 %}
16830 
16831 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16832 %{
16833   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16834   effect(KILL temp, KILL result);
16835 
16836   ins_cost(1100);  // slightly larger than the next version
16837   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16838 
16839   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16840 
16841   opcode(0x0); // Don't zero result reg on hit
16842 
16843   ins_pipe(pipe_class_memory);
16844 %}
16845 
16846 // Intrisics for String.compareTo()
16847 
16848 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16849                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16850 %{
16851   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16852   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16853   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16854 
16855   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16856   ins_encode %{
16857     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16858     __ string_compare($str1$$Register, $str2$$Register,
16859                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16860                       $tmp1$$Register, $tmp2$$Register,
16861                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16862   %}
16863   ins_pipe(pipe_class_memory);
16864 %}
16865 
16866 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16867                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16868 %{
16869   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16870   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16871   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16872 
16873   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16874   ins_encode %{
16875     __ string_compare($str1$$Register, $str2$$Register,
16876                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16877                       $tmp1$$Register, $tmp2$$Register,
16878                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16879   %}
16880   ins_pipe(pipe_class_memory);
16881 %}
16882 
16883 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16884                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16885                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16886 %{
16887   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16888   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16889   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16890          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16891 
16892   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16893   ins_encode %{
16894     __ string_compare($str1$$Register, $str2$$Register,
16895                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16896                       $tmp1$$Register, $tmp2$$Register,
16897                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16898                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16899   %}
16900   ins_pipe(pipe_class_memory);
16901 %}
16902 
16903 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16904                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16905                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16906 %{
16907   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16908   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16909   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16910          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16911 
16912   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16913   ins_encode %{
16914     __ string_compare($str1$$Register, $str2$$Register,
16915                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16916                       $tmp1$$Register, $tmp2$$Register,
16917                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16918                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16919   %}
16920   ins_pipe(pipe_class_memory);
16921 %}
16922 
16923 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16924 // these string_compare variants as NEON register type for convenience so that the prototype of
16925 // string_compare can be shared with all variants.
16926 
16927 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16928                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16929                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16930                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16931 %{
16932   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16933   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16934   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16935          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16936 
16937   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16938   ins_encode %{
16939     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16940     __ string_compare($str1$$Register, $str2$$Register,
16941                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16942                       $tmp1$$Register, $tmp2$$Register,
16943                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16944                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16945                       StrIntrinsicNode::LL);
16946   %}
16947   ins_pipe(pipe_class_memory);
16948 %}
16949 
16950 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16951                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16952                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16953                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16954 %{
16955   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16956   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16957   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16958          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16959 
16960   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16961   ins_encode %{
16962     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16963     __ string_compare($str1$$Register, $str2$$Register,
16964                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16965                       $tmp1$$Register, $tmp2$$Register,
16966                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16967                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16968                       StrIntrinsicNode::LU);
16969   %}
16970   ins_pipe(pipe_class_memory);
16971 %}
16972 
16973 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16974                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16975                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16976                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16977 %{
16978   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16979   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16980   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16981          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16982 
16983   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16984   ins_encode %{
16985     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16986     __ string_compare($str1$$Register, $str2$$Register,
16987                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16988                       $tmp1$$Register, $tmp2$$Register,
16989                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16990                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16991                       StrIntrinsicNode::UL);
16992   %}
16993   ins_pipe(pipe_class_memory);
16994 %}
16995 
16996 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16997                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16998                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16999                               pRegGov_P1 pgtmp2, rFlagsReg cr)
17000 %{
17001   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17002   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17003   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17004          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17005 
17006   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
17007   ins_encode %{
17008     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17009     __ string_compare($str1$$Register, $str2$$Register,
17010                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
17011                       $tmp1$$Register, $tmp2$$Register,
17012                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17013                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17014                       StrIntrinsicNode::UU);
17015   %}
17016   ins_pipe(pipe_class_memory);
17017 %}
17018 
17019 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17020        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17021        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17022 %{
17023   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17024   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17025   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17026          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17027   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
17028 
17029   ins_encode %{
17030     __ string_indexof($str1$$Register, $str2$$Register,
17031                       $cnt1$$Register, $cnt2$$Register,
17032                       $tmp1$$Register, $tmp2$$Register,
17033                       $tmp3$$Register, $tmp4$$Register,
17034                       $tmp5$$Register, $tmp6$$Register,
17035                       -1, $result$$Register, StrIntrinsicNode::UU);
17036   %}
17037   ins_pipe(pipe_class_memory);
17038 %}
17039 
17040 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17041        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17042        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17043 %{
17044   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17045   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17046   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17047          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17048   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17049 
17050   ins_encode %{
17051     __ string_indexof($str1$$Register, $str2$$Register,
17052                       $cnt1$$Register, $cnt2$$Register,
17053                       $tmp1$$Register, $tmp2$$Register,
17054                       $tmp3$$Register, $tmp4$$Register,
17055                       $tmp5$$Register, $tmp6$$Register,
17056                       -1, $result$$Register, StrIntrinsicNode::LL);
17057   %}
17058   ins_pipe(pipe_class_memory);
17059 %}
17060 
17061 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17062        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17063        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17064 %{
17065   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17066   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17067   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17068          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17069   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17070 
17071   ins_encode %{
17072     __ string_indexof($str1$$Register, $str2$$Register,
17073                       $cnt1$$Register, $cnt2$$Register,
17074                       $tmp1$$Register, $tmp2$$Register,
17075                       $tmp3$$Register, $tmp4$$Register,
17076                       $tmp5$$Register, $tmp6$$Register,
17077                       -1, $result$$Register, StrIntrinsicNode::UL);
17078   %}
17079   ins_pipe(pipe_class_memory);
17080 %}
17081 
17082 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17083                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17084                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17085 %{
17086   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17087   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17088   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17089          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17090   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17091 
17092   ins_encode %{
17093     int icnt2 = (int)$int_cnt2$$constant;
17094     __ string_indexof($str1$$Register, $str2$$Register,
17095                       $cnt1$$Register, zr,
17096                       $tmp1$$Register, $tmp2$$Register,
17097                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17098                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17099   %}
17100   ins_pipe(pipe_class_memory);
17101 %}
17102 
17103 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17104                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17105                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17106 %{
17107   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17108   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17109   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17110          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17111   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17112 
17113   ins_encode %{
17114     int icnt2 = (int)$int_cnt2$$constant;
17115     __ string_indexof($str1$$Register, $str2$$Register,
17116                       $cnt1$$Register, zr,
17117                       $tmp1$$Register, $tmp2$$Register,
17118                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17119                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17120   %}
17121   ins_pipe(pipe_class_memory);
17122 %}
17123 
17124 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17125                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17126                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17127 %{
17128   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17129   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17130   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17131          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17132   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17133 
17134   ins_encode %{
17135     int icnt2 = (int)$int_cnt2$$constant;
17136     __ string_indexof($str1$$Register, $str2$$Register,
17137                       $cnt1$$Register, zr,
17138                       $tmp1$$Register, $tmp2$$Register,
17139                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17140                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17141   %}
17142   ins_pipe(pipe_class_memory);
17143 %}
17144 
17145 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17146                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17147                              iRegINoSp tmp3, rFlagsReg cr)
17148 %{
17149   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17150   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17151   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17152          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17153 
17154   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17155 
17156   ins_encode %{
17157     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17158                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17159                            $tmp3$$Register);
17160   %}
17161   ins_pipe(pipe_class_memory);
17162 %}
17163 
17164 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17165                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17166                               iRegINoSp tmp3, rFlagsReg cr)
17167 %{
17168   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17169   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17170   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17171          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17172 
17173   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17174 
17175   ins_encode %{
17176     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17177                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17178                             $tmp3$$Register);
17179   %}
17180   ins_pipe(pipe_class_memory);
17181 %}
17182 
17183 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17184                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17185                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17186   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17187   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17188   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17189   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17190   ins_encode %{
17191     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17192                                $result$$Register, $ztmp1$$FloatRegister,
17193                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17194                                $ptmp$$PRegister, true /* isL */);
17195   %}
17196   ins_pipe(pipe_class_memory);
17197 %}
17198 
17199 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17200                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17201                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17202   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17203   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17204   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17205   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17206   ins_encode %{
17207     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17208                                $result$$Register, $ztmp1$$FloatRegister,
17209                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17210                                $ptmp$$PRegister, false /* isL */);
17211   %}
17212   ins_pipe(pipe_class_memory);
17213 %}
17214 
17215 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17216                         iRegI_R0 result, rFlagsReg cr)
17217 %{
17218   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17219   match(Set result (StrEquals (Binary str1 str2) cnt));
17220   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17221 
17222   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17223   ins_encode %{
17224     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17225     __ string_equals($str1$$Register, $str2$$Register,
17226                      $result$$Register, $cnt$$Register, 1);
17227   %}
17228   ins_pipe(pipe_class_memory);
17229 %}
17230 
17231 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17232                         iRegI_R0 result, rFlagsReg cr)
17233 %{
17234   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17235   match(Set result (StrEquals (Binary str1 str2) cnt));
17236   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17237 
17238   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17239   ins_encode %{
17240     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17241     __ string_equals($str1$$Register, $str2$$Register,
17242                      $result$$Register, $cnt$$Register, 2);
17243   %}
17244   ins_pipe(pipe_class_memory);
17245 %}
17246 
17247 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17248                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17249                        iRegP_R10 tmp, rFlagsReg cr)
17250 %{
17251   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17252   match(Set result (AryEq ary1 ary2));
17253   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17254 
17255   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17256   ins_encode %{
17257     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17258                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17259                                    $result$$Register, $tmp$$Register, 1);
17260     if (tpc == NULL) {
17261       ciEnv::current()->record_failure("CodeCache is full");
17262       return;
17263     }
17264   %}
17265   ins_pipe(pipe_class_memory);
17266 %}
17267 
17268 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17269                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17270                        iRegP_R10 tmp, rFlagsReg cr)
17271 %{
17272   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17273   match(Set result (AryEq ary1 ary2));
17274   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17275 
17276   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17277   ins_encode %{
17278     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17279                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17280                                    $result$$Register, $tmp$$Register, 2);
17281     if (tpc == NULL) {
17282       ciEnv::current()->record_failure("CodeCache is full");
17283       return;
17284     }
17285   %}
17286   ins_pipe(pipe_class_memory);
17287 %}
17288 
17289 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17290 %{
17291   match(Set result (CountPositives ary1 len));
17292   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17293   format %{ "count positives byte[] $ary1,$len -> $result" %}
17294   ins_encode %{
17295     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17296     if (tpc == NULL) {
17297       ciEnv::current()->record_failure("CodeCache is full");
17298       return;
17299     }
17300   %}
17301   ins_pipe( pipe_slow );
17302 %}
17303 
17304 // fast char[] to byte[] compression
17305 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17306                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17307                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17308                          iRegI_R0 result, rFlagsReg cr)
17309 %{
17310   match(Set result (StrCompressedCopy src (Binary dst len)));
17311   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17312          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17313 
17314   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17315   ins_encode %{
17316     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17317                            $result$$Register,
17318                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17319                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17320   %}
17321   ins_pipe(pipe_slow);
17322 %}
17323 
17324 // fast byte[] to char[] inflation
17325 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17326                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17327 %{
17328   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17329   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17330 
17331   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17332   ins_encode %{
17333     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17334                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17335                                         $tmp3$$FloatRegister, $tmp4$$Register);
17336     if (tpc == NULL) {
17337       ciEnv::current()->record_failure("CodeCache is full");
17338       return;
17339     }
17340   %}
17341   ins_pipe(pipe_class_memory);
17342 %}
17343 
17344 // encode char[] to byte[] in ISO_8859_1
17345 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17346                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17347                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17348                           iRegI_R0 result, rFlagsReg cr)
17349 %{
17350   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17351   match(Set result (EncodeISOArray src (Binary dst len)));
17352   effect(USE_KILL src, USE_KILL dst, USE len,
17353          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17354 
17355   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17356   ins_encode %{
17357     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17358                         $result$$Register, false,
17359                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17360                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17361   %}
17362   ins_pipe(pipe_class_memory);
17363 %}
17364 
17365 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17366                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17367                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17368                             iRegI_R0 result, rFlagsReg cr)
17369 %{
17370   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17371   match(Set result (EncodeISOArray src (Binary dst len)));
17372   effect(USE_KILL src, USE_KILL dst, USE len,
17373          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17374 
17375   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17376   ins_encode %{
17377     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17378                         $result$$Register, true,
17379                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17380                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17381   %}
17382   ins_pipe(pipe_class_memory);
17383 %}
17384 
17385 // ============================================================================
17386 // This name is KNOWN by the ADLC and cannot be changed.
17387 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17388 // for this guy.
17389 instruct tlsLoadP(thread_RegP dst)
17390 %{
17391   match(Set dst (ThreadLocal));
17392 
17393   ins_cost(0);
17394 
17395   format %{ " -- \t// $dst=Thread::current(), empty" %}
17396 
17397   size(0);
17398 
17399   ins_encode( /*empty*/ );
17400 
17401   ins_pipe(pipe_class_empty);
17402 %}
17403 
17404 //----------PEEPHOLE RULES-----------------------------------------------------
17405 // These must follow all instruction definitions as they use the names
17406 // defined in the instructions definitions.
17407 //
17408 // peepmatch ( root_instr_name [preceding_instruction]* );
17409 //
17410 // peepconstraint %{
17411 // (instruction_number.operand_name relational_op instruction_number.operand_name
17412 //  [, ...] );
17413 // // instruction numbers are zero-based using left to right order in peepmatch
17414 //
17415 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17416 // // provide an instruction_number.operand_name for each operand that appears
17417 // // in the replacement instruction's match rule
17418 //
17419 // ---------VM FLAGS---------------------------------------------------------
17420 //
17421 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17422 //
17423 // Each peephole rule is given an identifying number starting with zero and
17424 // increasing by one in the order seen by the parser.  An individual peephole
17425 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17426 // on the command-line.
17427 //
17428 // ---------CURRENT LIMITATIONS----------------------------------------------
17429 //
17430 // Only match adjacent instructions in same basic block
17431 // Only equality constraints
17432 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17433 // Only one replacement instruction
17434 //
17435 // ---------EXAMPLE----------------------------------------------------------
17436 //
17437 // // pertinent parts of existing instructions in architecture description
17438 // instruct movI(iRegINoSp dst, iRegI src)
17439 // %{
17440 //   match(Set dst (CopyI src));
17441 // %}
17442 //
17443 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17444 // %{
17445 //   match(Set dst (AddI dst src));
17446 //   effect(KILL cr);
17447 // %}
17448 //
17449 // // Change (inc mov) to lea
17450 // peephole %{
17451 //   // increment preceded by register-register move
17452 //   peepmatch ( incI_iReg movI );
17453 //   // require that the destination register of the increment
17454 //   // match the destination register of the move
17455 //   peepconstraint ( 0.dst == 1.dst );
17456 //   // construct a replacement instruction that sets
17457 //   // the destination to ( move's source register + one )
17458 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17459 // %}
17460 //
17461 
17462 // Implementation no longer uses movX instructions since
17463 // machine-independent system no longer uses CopyX nodes.
17464 //
17465 // peephole
17466 // %{
17467 //   peepmatch (incI_iReg movI);
17468 //   peepconstraint (0.dst == 1.dst);
17469 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17470 // %}
17471 
17472 // peephole
17473 // %{
17474 //   peepmatch (decI_iReg movI);
17475 //   peepconstraint (0.dst == 1.dst);
17476 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17477 // %}
17478 
17479 // peephole
17480 // %{
17481 //   peepmatch (addI_iReg_imm movI);
17482 //   peepconstraint (0.dst == 1.dst);
17483 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17484 // %}
17485 
17486 // peephole
17487 // %{
17488 //   peepmatch (incL_iReg movL);
17489 //   peepconstraint (0.dst == 1.dst);
17490 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17491 // %}
17492 
17493 // peephole
17494 // %{
17495 //   peepmatch (decL_iReg movL);
17496 //   peepconstraint (0.dst == 1.dst);
17497 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17498 // %}
17499 
17500 // peephole
17501 // %{
17502 //   peepmatch (addL_iReg_imm movL);
17503 //   peepconstraint (0.dst == 1.dst);
17504 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17505 // %}
17506 
17507 // peephole
17508 // %{
17509 //   peepmatch (addP_iReg_imm movP);
17510 //   peepconstraint (0.dst == 1.dst);
17511 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17512 // %}
17513 
17514 // // Change load of spilled value to only a spill
17515 // instruct storeI(memory mem, iRegI src)
17516 // %{
17517 //   match(Set mem (StoreI mem src));
17518 // %}
17519 //
17520 // instruct loadI(iRegINoSp dst, memory mem)
17521 // %{
17522 //   match(Set dst (LoadI mem));
17523 // %}
17524 //
17525 
17526 //----------SMARTSPILL RULES---------------------------------------------------
17527 // These must follow all instruction definitions as they use the names
17528 // defined in the instructions definitions.
17529 
17530 // Local Variables:
17531 // mode: c++
17532 // End:
--- EOF ---